text_classification_2/htk180000_text-classification-2.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Portfolio Assginment: Text Classification 2\n\nCS 4395.002 Human Language Technologies\n\nDr. Mazidi\n\nHenry Kim HTK180000\n\n## Overview\nThis Kaggle notebook uses deep learning approaches to classify headlines from the [Clickbait Dataset](https://www.kaggle.com/datasets/amananandrai/clickbait-dataset) as either clickbait or not clickbait.\n\nhttps://www.kaggle.com/datasets/amananandrai/clickbait-dataset\n\nThe dataset has 32,000 observations. Each observation has the article headline and the classification of 1 for clickbait and 0 for not clickbait.","metadata":{}},{"cell_type":"code","source":"# Import the libraries\nimport pandas as pd\nimport numpy as np\nimport re\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nimport keras\n\n# Load the train and test data \ndataset = pd.read_csv('../input/clickbait-dataset/clickbait_data.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:44.723666Z","iopub.execute_input":"2023-04-20T07:32:44.724218Z","iopub.status.idle":"2023-04-20T07:32:44.804421Z","shell.execute_reply.started":"2023-04-20T07:32:44.724176Z","shell.execute_reply":"2023-04-20T07:32:44.803264Z"},"trusted":true},"execution_count":77,"outputs":[]},{"cell_type":"code","source":"# Display some stats about the data and some datapoints\nprint('Shape: ' + str(dataset.shape) + '\\n')\nprint('Classification ({}):'.format(len(dataset['clickbait'].unique())))\nprint(dataset['clickbait'].unique())\nprint()\nprint(dataset.head())","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:44.806434Z","iopub.execute_input":"2023-04-20T07:32:44.806774Z","iopub.status.idle":"2023-04-20T07:32:44.818983Z","shell.execute_reply.started":"2023-04-20T07:32:44.806742Z","shell.execute_reply":"2023-04-20T07:32:44.817973Z"},"trusted":true},"execution_count":78,"outputs":[{"name":"stdout","text":"Shape: (32000, 2)\n\nClassification (2):\n[1 0]\n\n                                            headline  clickbait\n0                                 Should I Get Bings          1\n1      Which TV Female Friend Group Do You Belong In          1\n2  The New \"Star Wars: The Force Awakens\" Trailer...          1\n3  This Vine Of New York On \"Celebrity Big Brothe...          1\n4  A Couple Did A Stunning Photo Shoot With Their...          1\n","output_type":"stream"}]},{"cell_type":"markdown","source":"The data is evenly split between examples of clickbait and non-clickbait headlines.","metadata":{}},{"cell_type":"code","source":"graph = sns.countplot(data=dataset, x='clickbait')\ngraph.set_xticklabels(graph.get_xticklabels())\nplt.title(\"Count of Categories\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:44.820176Z","iopub.execute_input":"2023-04-20T07:32:44.820905Z","iopub.status.idle":"2023-04-20T07:32:45.038366Z","shell.execute_reply.started":"2023-04-20T07:32:44.820868Z","shell.execute_reply":"2023-04-20T07:32:45.037141Z"},"trusted":true},"execution_count":79,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAk0AAAHFCAYAAADv8c1wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAB0lEQVR4nO3de1hVZd7/8c+Wk2iy5CDQLjSbMVIhc7AQrbQw1AKyprBodjnjaBOTDomHnEZNZ9JRM21yNHUs8xQ9M6aNNTFYqeV4ptBItMPPAicQS9wIGhCu3x89rsctpkuS2Nj7dV37ulz3+q61v/f2Mj7da+2FwzRNUwAAADirFk3dAAAAQHNAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCfkR2796tX/7yl+rYsaNatmypSy65RD/72c80Y8YMHT58uKnbkyStXLlSc+bMaZRz/+EPf1D79u3l6+urtm3bnrP+3XffVVpami677DL5+/vLMAz16tVL8+fPV1VV1Xm/f2POrTFs2LBBDodDGzZsaOpWAK/g4NeoAD8OixYtUkZGhqKjo5WRkaEuXbqotrZWO3fu1KJFi9StWzetXr26qdtUcnKyCgoK9Nlnn13Q87766qsaNGiQHn/8cQ0cOFABAQHq0aPHd9ZPmjRJU6ZMUa9evTR06FD95Cc/0bFjx7R582YtXLhQ6enpmj179nn10FhzaywVFRXas2ePunTpoqCgoKZuB2hyvk3dAIDGt2XLFj388MO69dZbtWbNGgUEBFj7br31VmVlZSknJ6cJO2x8BQUFkqSRI0cqPDz8rLV///vfNWXKFA0dOlSLFi2Sw+Gw9g0cOFBjx47Vli1bGrXfplRbWyuHw6GgoCD17NmzqdsBvIcJ4KKXnJxs+vr6mkVFRbbq6+rqzOnTp5vR0dGmv7+/2a5dO9PlcpnFxcUedR06dDAffPDBesf36dPH7NOnj7W9fv16U5K5cuVK8/e//7156aWXmm3atDETExPNvXv3ehwnqd7r+/baoUOHeuecNGnSd54zJibGDA4ONquqqs7+Qf2vuXPnmjfeeKPZrl07s1WrVmZMTIw5ffp0s6amxvbcqqurzT/+8Y/WPMLCwswhQ4aYZWVlHu/19ddfm6NGjTIjIiLMwMBA88YbbzR37tx5xr+LDz74wExNTTXbtm1rBgQEmN26dTOXLFniUXPy72bp0qXmqFGjTKfTaTocDrOwsNDat379eo9jduzYYaakpJjBwcFmQECAee2115ovv/yyR01VVZWZlZVlXnHFFWZAQIAZHBxsxsXFmStXrrT1mQLeiJUm4CJXV1ent99+W3FxcYqKirJ1zMMPP6yFCxfqkUceUXJysj777DNNmDBBGzZs0HvvvaewsLAG9fL73/9evXv31t/+9jdVVFRo3LhxSklJUWFhoXx8fDRv3jwNHz5cn376qe1LhXZ6Xb16tf76179q8eLFysnJkWEYuvzyy894vpKSEhUUFGjw4MFq1aqVrR4+/fRTpaenq2PHjvL399euXbv05JNPau/evXr++ecl6axzO3HihO644w69++67Gjt2rHr16qXPP/9ckyZNUt++fbVz504FBgZKkn75y1/q5Zdf1tixY3XLLbdoz549uvPOO1VRUeFxzn379qlXr14KDw/XX/7yF4WGhmr58uUaMmSIDh48qLFjx3rUjx8/XgkJCXruuefUokULhYeHq7S0tN5c169frwEDBig+Pl7PPfecDMNQdna2Bg8erGPHjmnIkCGSpFGjRmnZsmX605/+pO7du6uqqkoFBQX66quvbH2mgFdq6tQGoHGVlpaaksx7773XVn1hYaEpyczIyPAY37ZtmynJ/P3vf2+Nne9K02233eZR9z//8z+mJHPLli3W2O2332526NDhgvc6adIkU5J56NChs55z69atpiTzscces9XD6erq6sza2lpz6dKlpo+Pj3n48GFr33fN7aWXXjIlmatWrfIY37FjhynJnDdvnmmapvnhhx+aksxx48ad8fhT/y7uvfdeMyAgoN7q4sCBA81WrVqZR44cMU3z//5ubrrppnp9nWml6eqrrza7d+9u1tbWetQmJyebl156qVlXV2ea5rerdYMGDfqOTwlonvj2HAAP69evlyRrxeCk66+/Xp07d9Zbb73V4HOnpqZ6bF9zzTWSpM8//7xB52vMXs/H+++/r9TUVIWGhsrHx0d+fn564IEHVFdXp48++uicx7/22mtq27atUlJS9M0331iva6+9VpGRkda31zZu3ChJSktL8zj+7rvvlq+v54WDt99+W4mJifVWF4cMGaJjx47Vuyfr5z//+Tn7/OSTT7R3717df//9kuTR62233aaSkhLt27dP0rd/B2+88YYee+wxbdiwQcePHz/n+QFvR2gCLnJhYWFq1aqV9u/fb6v+5OWTSy+9tN4+p9P5vS6vhIaGemyfvCG9oT9QG6PX9u3bS5Ltz6uoqEg33nij/vvf/+qZZ57Ru+++qx07duivf/2rJHtzO3jwoI4cOSJ/f3/5+fl5vEpLS/Xll19K+r/5RkREeBzv6+tb77P96quvvvNzOfVcJ52p9kx9StLo0aPr9ZmRkSFJVq9/+ctfNG7cOK1Zs0Y333yzQkJCNGjQIH388cfnfB/AW3FPE3CR8/HxUWJiot544w0dOHDgO+/lOenkD9+SkpJ6tV988YXH/UwtW7ZUdXV1vXN8+eWXDb7v6XycT692XXrppYqNjVVubq6OHTt2zvua1qxZo6qqKr3yyivq0KGDNZ6fn2/7PcPCwhQaGvqd32Bs06aNpP+b78GDB3XZZZdZ+7/55pt6ISg0NFQlJSX1zvXFF19Y73mqU78heLY+pW/vf7rrrrvOWBMdHS1Jat26tSZPnqzJkyfr4MGD1qpTSkqK9u7de873ArwRK03Aj8D48eNlmqaGDRummpqaevtra2u1du1aSdItt9wiSVq+fLlHzY4dO1RYWKjExERr7IorrtDu3bs96j766CPrEk1DBAQE2F55Op9ez8eECRNUXl6ukSNHyjzDo+wqKyuVm5sr6f/CxqmPcTBNU4sWLap33HfNLTk5WV999ZXq6urUo0ePeq+TQeSmm26SJL388ssex//jH//QN9984zGWmJiot99+2wpJJy1dulStWrVq0KMEoqOj1alTJ+3ateuMffbo0cMKeKeKiIjQkCFDdN9992nfvn06duzYeb834A1YaQJ+BBISEjR//nxlZGQoLi5ODz/8sLp27ara2lq9//77WrhwoWJiYpSSkqLo6GgNHz5czz77rFq0aKGBAwda30iLiorSo48+ap3X5XLpF7/4hTIyMvTzn/9cn3/+uWbMmKF27do1uNfY2Fi98sormj9/vuLi4tSiRYvvfAjl+fR6Pu655x5NmDBBf/zjH7V3716Ph1tu27ZNCxYs0ODBg5WUlKRbb71V/v7+uu+++zR27Fh9/fXXmj9/vsrLy23P7d5779WKFSt022236Xe/+52uv/56+fn56cCBA1q/fr3uuOMO3Xnnneratavuu+8+zZo1Sz4+Prrlllv04YcfatasWTIMQy1a/N//B0+aNEmvvfaabr75Zk2cOFEhISFasWKFXn/9dc2YMUOGYTTos1mwYIEGDhyo/v37a8iQIbrssst0+PBhFRYW6r333tPf//53SVJ8fLySk5N1zTXXKDg4WIWFhVq2bJkSEhJsfysR8DpNfCM6gB9Qfn6++eCDD5rt27c3/f39zdatW5vdu3c3J06c6PE8oJPPPrrqqqtMPz8/MywszPzFL35R7zlNJ06cMGfMmGFeeeWVZsuWLc0ePXqYb7/99nd+e+7vf/+7x/H79+83JZkvvPCCNXb48GHz7rvvNtu2bWs6HA7bz2k6V692vz13qo0bN5p33323eemll5p+fn5mUFCQmZCQYM6cOdOsqKiw6tauXWt269bNbNmypXnZZZeZY8aMMd9444163zw729xqa2vNp556yjrPJZdcYl599dXmQw89ZH788cdW3cnnNIWHh5stW7Y0e/bsaW7ZssU0DMN89NFHPfr/4IMPzJSUFNMwDNPf39/s1q2bx2dtmt/9d3PqvtOf07Rr1y4zLS3NDA8PN/38/MzIyEjzlltuMZ977jmr5rHHHjN79OhhPcvpyiuvNB999FHzyy+/tP35A96GX6MCAM3c5s2b1bt3b61YsULp6elN3Q5w0SI0AUAzsm7dOm3ZskVxcXEKDAzUrl279Oc//1mGYWj37t1q2bJlU7cIXLS4pwkAmpGgoCDl5uZqzpw5Onr0qMLCwjRw4EBNmzaNwAQ0MlaaAAAAbOCRAwAAADYQmgAAAGwgNAEAANjAjeAX0IkTJ/TFF1+oTZs2tn4lAQAAaHqmaero0aNyOp0eD4k9HaHpAvriiy/q/UZxAADQPBQXF5/193MSmi6gk79zqbi4WEFBQU3cDQAAsKOiokJRUVFn/N2JpyI0XUAnL8kFBQURmgAAaGbOdWsNN4IDAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsKFJQ9M777yjlJQUOZ1OORwOrVmzpl5NYWGhUlNTZRiG2rRpo549e6qoqMjaX11drREjRigsLEytW7dWamqqDhw44HGO8vJyuVwuGYYhwzDkcrl05MgRj5qioiKlpKSodevWCgsL08iRI1VTU9MY0wYAAM1Qk4amqqoqdevWTXPnzj3j/k8//VQ33HCDrr76am3YsEG7du3ShAkT1LJlS6smMzNTq1evVnZ2tjZt2qTKykolJyerrq7OqklPT1d+fr5ycnKUk5Oj/Px8uVwua39dXZ1uv/12VVVVadOmTcrOztaqVauUlZXVeJMHAADNisM0TbOpm5C+/SV5q1ev1qBBg6yxe++9V35+flq2bNkZj3G73WrXrp2WLVumwYMHS5K++OILRUVF6V//+pf69++vwsJCdenSRVu3blV8fLwkaevWrUpISNDevXsVHR2tN954Q8nJySouLpbT6ZQkZWdna8iQISorK7P9y3crKipkGIbcbje/sBcAgGbC7s9vr72n6cSJE3r99dd11VVXqX///goPD1d8fLzHJby8vDzV1tYqKSnJGnM6nYqJidHmzZslSVu2bJFhGFZgkqSePXvKMAyPmpiYGCswSVL//v1VXV2tvLy8Rp4pAABoDrw2NJWVlamyslJ//vOfNWDAAOXm5urOO+/UXXfdpY0bN0qSSktL5e/vr+DgYI9jIyIiVFpaatWEh4fXO394eLhHTUREhMf+4OBg+fv7WzVnUl1drYqKCo8XAAC4OPk2dQPf5cSJE5KkO+64Q48++qgk6dprr9XmzZv13HPPqU+fPt95rGmacjgc1vapf/4+NaebNm2aJk+efO7JXEBxY5b+oO8HNBd5Mx9o6ha+t6IpsU3dAuCV2k/8oKlbkOTFK01hYWHy9fVVly5dPMY7d+5sfXsuMjJSNTU1Ki8v96gpKyuzVo4iIyN18ODBeuc/dOiQR83pK0rl5eWqra2ttwJ1qvHjx8vtdluv4uLi858oAABoFrw2NPn7++u6667Tvn37PMY/+ugjdejQQZIUFxcnPz8/rVu3ztpfUlKigoIC9erVS5KUkJAgt9ut7du3WzXbtm2T2+32qCkoKFBJSYlVk5ubq4CAAMXFxX1njwEBAQoKCvJ4AQCAi1OTXp6rrKzUJ598Ym3v379f+fn5CgkJUfv27TVmzBgNHjxYN910k26++Wbl5ORo7dq12rBhgyTJMAwNHTpUWVlZCg0NVUhIiEaPHq3Y2Fj169dP0rcrUwMGDNCwYcO0YMECSdLw4cOVnJys6OhoSVJSUpK6dOkil8ulmTNn6vDhwxo9erSGDRtGEAIAAJKaeKVp586d6t69u7p37y5JGjVqlLp3766JEydKku68804999xzmjFjhmJjY/W3v/1Nq1at0g033GCdY/bs2Ro0aJDS0tLUu3dvtWrVSmvXrpWPj49Vs2LFCsXGxiopKUlJSUm65pprPB5j4OPjo9dff10tW7ZU7969lZaWpkGDBumpp576gT4JAADg7bzmOU0Xgx/iOU3cCA6cGTeCAxevxr4RvNk/pwkAAMCbEJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANjRpaHrnnXeUkpIip9Mph8OhNWvWfGftQw89JIfDoTlz5niMV1dXa8SIEQoLC1Pr1q2VmpqqAwcOeNSUl5fL5XLJMAwZhiGXy6UjR4541BQVFSklJUWtW7dWWFiYRo4cqZqamgs0UwAA0Nw1aWiqqqpSt27dNHfu3LPWrVmzRtu2bZPT6ay3LzMzU6tXr1Z2drY2bdqkyspKJScnq66uzqpJT09Xfn6+cnJylJOTo/z8fLlcLmt/XV2dbr/9dlVVVWnTpk3Kzs7WqlWrlJWVdeEmCwAAmjXfpnzzgQMHauDAgWet+e9//6tHHnlE//73v3X77bd77HO73Vq8eLGWLVumfv36SZKWL1+uqKgovfnmm+rfv78KCwuVk5OjrVu3Kj4+XpK0aNEiJSQkaN++fYqOjlZubq727Nmj4uJiK5jNmjVLQ4YM0ZNPPqmgoKBGmD0AAGhOvPqephMnTsjlcmnMmDHq2rVrvf15eXmqra1VUlKSNeZ0OhUTE6PNmzdLkrZs2SLDMKzAJEk9e/aUYRgeNTExMR4rWf3791d1dbXy8vIaa3oAAKAZadKVpnOZPn26fH19NXLkyDPuLy0tlb+/v4KDgz3GIyIiVFpaatWEh4fXOzY8PNyjJiIiwmN/cHCw/P39rZozqa6uVnV1tbVdUVFhb2IAAKDZ8dqVpry8PD3zzDNasmSJHA7HeR1rmqbHMWc6viE1p5s2bZp1c7lhGIqKijqvPgEAQPPhtaHp3XffVVlZmdq3by9fX1/5+vrq888/V1ZWlq644gpJUmRkpGpqalReXu5xbFlZmbVyFBkZqYMHD9Y7/6FDhzxqTl9RKi8vV21tbb0VqFONHz9ebrfbehUXF3+fKQMAAC/mtaHJ5XJp9+7dys/Pt15Op1NjxozRv//9b0lSXFyc/Pz8tG7dOuu4kpISFRQUqFevXpKkhIQEud1ubd++3arZtm2b3G63R01BQYFKSkqsmtzcXAUEBCguLu47ewwICFBQUJDHCwAAXJya9J6myspKffLJJ9b2/v37lZ+fr5CQELVv316hoaEe9X5+foqMjFR0dLQkyTAMDR06VFlZWQoNDVVISIhGjx6t2NhY69t0nTt31oABAzRs2DAtWLBAkjR8+HAlJydb50lKSlKXLl3kcrk0c+ZMHT58WKNHj9awYcMIQgAAQFITrzTt3LlT3bt3V/fu3SVJo0aNUvfu3TVx4kTb55g9e7YGDRqktLQ09e7dW61atdLatWvl4+Nj1axYsUKxsbFKSkpSUlKSrrnmGi1btsza7+Pjo9dff10tW7ZU7969lZaWpkGDBumpp566cJMFAADNmsM0TbOpm7hYVFRUyDAMud3uRluhihuztFHOCzR3eTMfaOoWvreiKbFN3QLgldpP/KBRz2/357fX3tMEAADgTQhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABuaNDS98847SklJkdPplMPh0Jo1a6x9tbW1GjdunGJjY9W6dWs5nU498MAD+uKLLzzOUV1drREjRigsLEytW7dWamqqDhw44FFTXl4ul8slwzBkGIZcLpeOHDniUVNUVKSUlBS1bt1aYWFhGjlypGpqahpr6gAAoJlp0tBUVVWlbt26ae7cufX2HTt2TO+9954mTJig9957T6+88oo++ugjpaametRlZmZq9erVys7O1qZNm1RZWank5GTV1dVZNenp6crPz1dOTo5ycnKUn58vl8tl7a+rq9Ptt9+uqqoqbdq0SdnZ2Vq1apWysrIab/IAAKBZ8W3KNx84cKAGDhx4xn2GYWjdunUeY88++6yuv/56FRUVqX379nK73Vq8eLGWLVumfv36SZKWL1+uqKgovfnmm+rfv78KCwuVk5OjrVu3Kj4+XpK0aNEiJSQkaN++fYqOjlZubq727Nmj4uJiOZ1OSdKsWbM0ZMgQPfnkkwoKCmrETwEAADQHzeqeJrfbLYfDobZt20qS8vLyVFtbq6SkJKvG6XQqJiZGmzdvliRt2bJFhmFYgUmSevbsKcMwPGpiYmKswCRJ/fv3V3V1tfLy8n6AmQEAAG/XpCtN5+Prr7/WY489pvT0dGvlp7S0VP7+/goODvaojYiIUGlpqVUTHh5e73zh4eEeNRERER77g4OD5e/vb9WcSXV1taqrq63tioqKhk0OAAB4vWax0lRbW6t7771XJ06c0Lx5885Zb5qmHA6HtX3qn79PzemmTZtm3VxuGIaioqLO2RsAAGievD401dbWKi0tTfv379e6des87i+KjIxUTU2NysvLPY4pKyuzVo4iIyN18ODBeuc9dOiQR83pK0rl5eWqra2ttwJ1qvHjx8vtdluv4uLiBs8TAAB4N68OTScD08cff6w333xToaGhHvvj4uLk5+fnccN4SUmJCgoK1KtXL0lSQkKC3G63tm/fbtVs27ZNbrfbo6agoEAlJSVWTW5urgICAhQXF/ed/QUEBCgoKMjjBQAALk5Nek9TZWWlPvnkE2t7//79ys/PV0hIiJxOp+6++2699957eu2111RXV2etBoWEhMjf31+GYWjo0KHKyspSaGioQkJCNHr0aMXGxlrfpuvcubMGDBigYcOGacGCBZKk4cOHKzk5WdHR0ZKkpKQkdenSRS6XSzNnztThw4c1evRoDRs2jCAEAAAkNXFo2rlzp26++WZre9SoUZKkBx98UE888YT++c9/SpKuvfZaj+PWr1+vvn37SpJmz54tX19fpaWl6fjx40pMTNSSJUvk4+Nj1a9YsUIjR460vmWXmprq8WwoHx8fvf7668rIyFDv3r0VGBio9PR0PfXUU40xbQAA0Aw5TNM0m7qJi0VFRYUMw5Db7W60Faq4MUsb5bxAc5c384GmbuF7K5oS29QtAF6p/cQPGvX8dn9+e/U9TQAAAN6C0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwoUlD0zvvvKOUlBQ5nU45HA6tWbPGY79pmnriiSfkdDoVGBiovn376sMPP/Soqa6u1ogRIxQWFqbWrVsrNTVVBw4c8KgpLy+Xy+WSYRgyDEMul0tHjhzxqCkqKlJKSopat26tsLAwjRw5UjU1NY0xbQAA0Aw1aWiqqqpSt27dNHfu3DPunzFjhp5++mnNnTtXO3bsUGRkpG699VYdPXrUqsnMzNTq1auVnZ2tTZs2qbKyUsnJyaqrq7Nq0tPTlZ+fr5ycHOXk5Cg/P18ul8vaX1dXp9tvv11VVVXatGmTsrOztWrVKmVlZTXe5AEAQLPi25RvPnDgQA0cOPCM+0zT1Jw5c/T444/rrrvukiS9+OKLioiI0MqVK/XQQw/J7XZr8eLFWrZsmfr16ydJWr58uaKiovTmm2+qf//+KiwsVE5OjrZu3ar4+HhJ0qJFi5SQkKB9+/YpOjpaubm52rNnj4qLi+V0OiVJs2bN0pAhQ/Tkk08qKCjoB/g0AACAN/Pae5r279+v0tJSJSUlWWMBAQHq06ePNm/eLEnKy8tTbW2tR43T6VRMTIxVs2XLFhmGYQUmSerZs6cMw/CoiYmJsQKTJPXv31/V1dXKy8tr1HkCAIDmoUlXms6mtLRUkhQREeExHhERoc8//9yq8ff3V3BwcL2ak8eXlpYqPDy83vnDw8M9ak5/n+DgYPn7+1s1Z1JdXa3q6mpru6Kiwu70AABAM+O1K00nORwOj23TNOuNne70mjPVN6TmdNOmTbNuLjcMQ1FRUWftCwAANF9eG5oiIyMlqd5KT1lZmbUqFBkZqZqaGpWXl5+15uDBg/XOf+jQIY+a09+nvLxctbW19VagTjV+/Hi53W7rVVxcfJ6zBAAAzYXXhqaOHTsqMjJS69ats8Zqamq0ceNG9erVS5IUFxcnPz8/j5qSkhIVFBRYNQkJCXK73dq+fbtVs23bNrndbo+agoIClZSUWDW5ubkKCAhQXFzcd/YYEBCgoKAgjxcAALg4Nek9TZWVlfrkk0+s7f379ys/P18hISFq3769MjMzNXXqVHXq1EmdOnXS1KlT1apVK6Wnp0uSDMPQ0KFDlZWVpdDQUIWEhGj06NGKjY21vk3XuXNnDRgwQMOGDdOCBQskScOHD1dycrKio6MlSUlJSerSpYtcLpdmzpypw4cPa/To0Ro2bBhBCAAASGri0LRz507dfPPN1vaoUaMkSQ8++KCWLFmisWPH6vjx48rIyFB5ebni4+OVm5urNm3aWMfMnj1bvr6+SktL0/Hjx5WYmKglS5bIx8fHqlmxYoVGjhxpfcsuNTXV49lQPj4+ev3115WRkaHevXsrMDBQ6enpeuqppxr7IwAAAM2EwzRNs6mbuFhUVFTIMAy53e5GW6GKG7O0Uc4LNHd5Mx9o6ha+t6IpsU3dAuCV2k/8oFHPb/fnt9fe0wQAAOBNCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2NCg0HTLLbfoyJEj9cYrKip0yy23fN+eAAAAvE6DQtOGDRtUU1NTb/zrr7/Wu++++72bAgAA8Da+51O8e/du68979uxRaWmptV1XV6ecnBxddtllF647AAAAL3Feoenaa6+Vw+GQw+E442W4wMBAPfvssxesOQAAAG9xXqFp//79Mk1TV155pbZv36527dpZ+/z9/RUeHi4fH58L3iQAAEBTO6/Q1KFDB0nSiRMnGqUZAAAAb3VeoelUH330kTZs2KCysrJ6IWrixInfuzEAAABv0qDQtGjRIj388MMKCwtTZGSkHA6Htc/hcBCaAADARadBoelPf/qTnnzySY0bN+5C9wMAAOCVGvScpvLyct1zzz0XuhcAAACv1aDQdM899yg3N/dC9wIAAOC1GnR57qc//akmTJigrVu3KjY2Vn5+fh77R44ceUGaAwAA8BYNCk0LFy7UJZdcoo0bN2rjxo0e+xwOB6EJAABcdBoUmvbv33+h+wAAAPBqDbqnCQAA4MemQStNv/rVr866//nnn29QMwAAAN6qQaGpvLzcY7u2tlYFBQU6cuTIGX+RLwAAQHPXoNC0evXqemMnTpxQRkaGrrzyyu/dFAAAgLe5YPc0tWjRQo8++qhmz559oU4JAADgNS7ojeCffvqpvvnmmwt5SgAAAK/QoMtzo0aN8tg2TVMlJSV6/fXX9eCDD16QxgAAALxJg1aa3n//fY/X7t27JUmzZs3SnDlzLlhz33zzjf7whz+oY8eOCgwM1JVXXqkpU6boxIkTVo1pmnriiSfkdDoVGBiovn376sMPP/Q4T3V1tUaMGKGwsDC1bt1aqampOnDggEdNeXm5XC6XDMOQYRhyuVw6cuTIBZsLAABo3hq00rR+/foL3ccZTZ8+Xc8995xefPFFde3aVTt37tQvf/lLGYah3/3ud5KkGTNm6Omnn9aSJUt01VVX6U9/+pNuvfVW7du3T23atJEkZWZmau3atcrOzlZoaKiysrKUnJysvLw8+fj4SJLS09N14MAB5eTkSJKGDx8ul8ultWvX/iBzBQAA3q1BoemkQ4cOad++fXI4HLrqqqvUrl27C9WXJGnLli264447dPvtt0uSrrjiCr300kvauXOnpG9XmebMmaPHH39cd911lyTpxRdfVEREhFauXKmHHnpIbrdbixcv1rJly9SvXz9J0vLlyxUVFaU333xT/fv3V2FhoXJycrR161bFx8dLkhYtWqSEhATt27dP0dHRF3ReAACg+WnQ5bmqqir96le/0qWXXqqbbrpJN954o5xOp4YOHapjx45dsOZuuOEGvfXWW/roo48kSbt27dKmTZt02223Sfr217mUlpYqKSnJOiYgIEB9+vTR5s2bJUl5eXmqra31qHE6nYqJibFqtmzZIsMwrMAkST179pRhGFYNAAD4cWtQaBo1apQ2btyotWvX6siRIzpy5IheffVVbdy4UVlZWResuXHjxum+++7T1VdfLT8/P3Xv3l2ZmZm67777JEmlpaWSpIiICI/jIiIirH2lpaXy9/dXcHDwWWvCw8PrvX94eLhVcybV1dWqqKjweAEAgItTgy7PrVq1Sv/4xz/Ut29fa+y2225TYGCg0tLSNH/+/AvS3Msvv6zly5dr5cqV6tq1q/Lz85WZmSmn0+nxLT2Hw+FxnGma9cZOd3rNmerPdZ5p06Zp8uTJdqcDAACasQatNB07dqze6o707crMhbw8N2bMGD322GO69957FRsbK5fLpUcffVTTpk2TJEVGRkpSvdWgsrIyq7/IyEjV1NTU+9Uvp9ccPHiw3vsfOnTojPM8afz48XK73daruLi44ZMFAABerUGhKSEhQZMmTdLXX39tjR0/flyTJ09WQkLCBWvu2LFjatHCs0UfHx/rkQMdO3ZUZGSk1q1bZ+2vqanRxo0b1atXL0lSXFyc/Pz8PGpKSkpUUFBg1SQkJMjtdmv79u1WzbZt2+R2u62aMwkICFBQUJDHCwAAXJwadHluzpw5GjhwoC6//HJ169ZNDodD+fn5CggIUG5u7gVrLiUlRU8++aTat2+vrl276v3339fTTz+tX/3qV5K+vaSWmZmpqVOnqlOnTurUqZOmTp2qVq1aKT09XZJkGIaGDh2qrKwshYaGKiQkRKNHj1ZsbKz1bbrOnTtrwIABGjZsmBYsWCDp20cOJCcn8805AAAgqYGhKTY2Vh9//LGWL1+uvXv3yjRN3Xvvvbr//vsVGBh4wZp79tlnNWHCBGVkZKisrExOp1MPPfSQJk6caNWMHTtWx48fV0ZGhsrLyxUfH6/c3FzrGU2SNHv2bPn6+iotLU3Hjx9XYmKilixZYj2jSZJWrFihkSNHWt+yS01N1dy5cy/YXAAAQPPmME3TPN+Dpk2bpoiICGvF56Tnn39ehw4d0rhx4y5Yg81JRUWFDMOQ2+1utEt1cWOWNsp5geYub+YDTd3C91Y0JbapWwC8UvuJHzTq+e3+/G7QPU0LFizQ1VdfXW+8a9eueu655xpySgAAAK/WoNBUWlqqSy+9tN54u3btVFJS8r2bAgAA8DYNCk1RUVH6z3/+U2/8P//5j5xO5/duCgAAwNs06EbwX//618rMzFRtba1uueUWSdJbb72lsWPHXtAnggMAAHiLBoWmsWPH6vDhw8rIyFBNTY0kqWXLlho3bpzGjx9/QRsEAADwBg0KTQ6HQ9OnT9eECRNUWFiowMBAderUSQEBARe6PwAAAK/QoNB00iWXXKLrrrvuQvUCAADgtRp0IzgAAMCPDaEJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABq8PTf/973/1i1/8QqGhoWrVqpWuvfZa5eXlWftN09QTTzwhp9OpwMBA9e3bVx9++KHHOaqrqzVixAiFhYWpdevWSk1N1YEDBzxqysvL5XK5ZBiGDMOQy+XSkSNHfogpAgCAZsCrQ1N5ebl69+4tPz8/vfHGG9qzZ49mzZqltm3bWjUzZszQ008/rblz52rHjh2KjIzUrbfeqqNHj1o1mZmZWr16tbKzs7Vp0yZVVlYqOTlZdXV1Vk16erry8/OVk5OjnJwc5efny+Vy/ZDTBQAAXsy3qRs4m+nTpysqKkovvPCCNXbFFVdYfzZNU3PmzNHjjz+uu+66S5L04osvKiIiQitXrtRDDz0kt9utxYsXa9myZerXr58kafny5YqKitKbb76p/v37q7CwUDk5Odq6davi4+MlSYsWLVJCQoL27dun6OjoH27SAADAK3n1StM///lP9ejRQ/fcc4/Cw8PVvXt3LVq0yNq/f/9+lZaWKikpyRoLCAhQnz59tHnzZklSXl6eamtrPWqcTqdiYmKsmi1btsgwDCswSVLPnj1lGIZVAwAAfty8OjT9v//3/zR//nx16tRJ//73v/Wb3/xGI0eO1NKlSyVJpaWlkqSIiAiP4yIiIqx9paWl8vf3V3Bw8FlrwsPD671/eHi4VXMm1dXVqqio8HgBAICLk1dfnjtx4oR69OihqVOnSpK6d++uDz/8UPPnz9cDDzxg1TkcDo/jTNOsN3a602vOVH+u80ybNk2TJ0+2NRcAANC8efVK06WXXqouXbp4jHXu3FlFRUWSpMjISEmqtxpUVlZmrT5FRkaqpqZG5eXlZ605ePBgvfc/dOhQvVWsU40fP15ut9t6FRcXn+cMAQBAc+HVoal3797at2+fx9hHH32kDh06SJI6duyoyMhIrVu3ztpfU1OjjRs3qlevXpKkuLg4+fn5edSUlJSooKDAqklISJDb7db27dutmm3btsntdls1ZxIQEKCgoCCPFwAAuDh59eW5Rx99VL169dLUqVOVlpam7du3a+HChVq4cKGkby+pZWZmaurUqerUqZM6deqkqVOnqlWrVkpPT5ckGYahoUOHKisrS6GhoQoJCdHo0aMVGxtrfZuuc+fOGjBggIYNG6YFCxZIkoYPH67k5GS+OQcAACR5eWi67rrrtHr1ao0fP15TpkxRx44dNWfOHN1///1WzdixY3X8+HFlZGSovLxc8fHxys3NVZs2baya2bNny9fXV2lpaTp+/LgSExO1ZMkS+fj4WDUrVqzQyJEjrW/Zpaamau7cuT/cZAEAgFdzmKZpNnUTF4uKigoZhiG3291ol+rixixtlPMCzV3ezAfOXeTliqbENnULgFdqP/GDRj2/3Z/fXn1PEwAAgLcgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsaFahadq0aXI4HMrMzLTGTNPUE088IafTqcDAQPXt21cffvihx3HV1dUaMWKEwsLC1Lp1a6WmpurAgQMeNeXl5XK5XDIMQ4ZhyOVy6ciRIz/ArAAAQHPQbELTjh07tHDhQl1zzTUe4zNmzNDTTz+tuXPnaseOHYqMjNStt96qo0ePWjWZmZlavXq1srOztWnTJlVWVio5OVl1dXVWTXp6uvLz85WTk6OcnBzl5+fL5XL9YPMDAADerVmEpsrKSt1///1atGiRgoODrXHTNDVnzhw9/vjjuuuuuxQTE6MXX3xRx44d08qVKyVJbrdbixcv1qxZs9SvXz91795dy5cv1wcffKA333xTklRYWKicnBz97W9/U0JCghISErRo0SK99tpr2rdvX5PMGQAAeJdmEZp++9vf6vbbb1e/fv08xvfv36/S0lIlJSVZYwEBAerTp482b94sScrLy1Ntba1HjdPpVExMjFWzZcsWGYah+Ph4q6Znz54yDMOqAQAAP26+Td3AuWRnZ+u9997Tjh076u0rLS2VJEVERHiMR0RE6PPPP7dq/P39PVaoTtacPL60tFTh4eH1zh8eHm7VnEl1dbWqq6ut7YqKCpuzAgAAzY1XrzQVFxfrd7/7nZYvX66WLVt+Z53D4fDYNk2z3tjpTq85U/25zjNt2jTrxnHDMBQVFXXW9wQAAM2XV4emvLw8lZWVKS4uTr6+vvL19dXGjRv1l7/8Rb6+vtYK0+mrQWVlZda+yMhI1dTUqLy8/Kw1Bw8erPf+hw4dqreKdarx48fL7XZbr+Li4u81XwAA4L28OjQlJibqgw8+UH5+vvXq0aOH7r//fuXn5+vKK69UZGSk1q1bZx1TU1OjjRs3qlevXpKkuLg4+fn5edSUlJSooKDAqklISJDb7db27dutmm3btsntdls1ZxIQEKCgoCCPFwAAuDh59T1Nbdq0UUxMjMdY69atFRoaao1nZmZq6tSp6tSpkzp16qSpU6eqVatWSk9PlyQZhqGhQ4cqKytLoaGhCgkJ0ejRoxUbG2vdWN65c2cNGDBAw4YN04IFCyRJw4cPV3JysqKjo3/AGQMAAG/l1aHJjrFjx+r48ePKyMhQeXm54uPjlZubqzZt2lg1s2fPlq+vr9LS0nT8+HElJiZqyZIl8vHxsWpWrFihkSNHWt+yS01N1dy5c3/w+QAAAO/kME3TbOomLhYVFRUyDENut7vRLtXFjVnaKOcFmru8mQ80dQvfW9GU2KZuAfBK7Sd+0Kjnt/vz26vvaQIAAPAWhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADV4dmqZNm6brrrtObdq0UXh4uAYNGqR9+/Z51JimqSeeeEJOp1OBgYHq27evPvzwQ4+a6upqjRgxQmFhYWrdurVSU1N14MABj5ry8nK5XC4ZhiHDMORyuXTkyJHGniIAAGgmvDo0bdy4Ub/97W+1detWrVu3Tt98842SkpJUVVVl1cyYMUNPP/205s6dqx07digyMlK33nqrjh49atVkZmZq9erVys7O1qZNm1RZWank5GTV1dVZNenp6crPz1dOTo5ycnKUn58vl8v1g84XAAB4L4dpmmZTN2HXoUOHFB4ero0bN+qmm26SaZpyOp3KzMzUuHHjJH27qhQREaHp06froYcektvtVrt27bRs2TINHjxYkvTFF18oKipK//rXv9S/f38VFhaqS5cu2rp1q+Lj4yVJW7duVUJCgvbu3avo6Ghb/VVUVMgwDLndbgUFBTXKZxA3ZmmjnBdo7vJmPtDULXxvRVNim7oFwCu1n/hBo57f7s9vr15pOp3b7ZYkhYSESJL279+v0tJSJSUlWTUBAQHq06ePNm/eLEnKy8tTbW2tR43T6VRMTIxVs2XLFhmGYQUmSerZs6cMw7BqAADAj5tvUzdgl2maGjVqlG644QbFxMRIkkpLSyVJERERHrURERH6/PPPrRp/f38FBwfXqzl5fGlpqcLDw+u9Z3h4uFVzJtXV1aqurra2KyoqGjAzAADQHDSblaZHHnlEu3fv1ksvvVRvn8Ph8Ng2TbPe2OlOrzlT/bnOM23aNOvGccMwFBUVda5pAACAZqpZhKYRI0bon//8p9avX6/LL7/cGo+MjJSkeqtBZWVl1upTZGSkampqVF5eftaagwcP1nvfQ4cO1VvFOtX48ePldrutV3FxccMmCAAAvJ5XhybTNPXII4/olVde0dtvv62OHTt67O/YsaMiIyO1bt06a6ympkYbN25Ur169JElxcXHy8/PzqCkpKVFBQYFVk5CQILfbre3bt1s127Ztk9vttmrOJCAgQEFBQR4vAABwcfLqe5p++9vfauXKlXr11VfVpk0ba0XJMAwFBgbK4XAoMzNTU6dOVadOndSpUydNnTpVrVq1Unp6ulU7dOhQZWVlKTQ0VCEhIRo9erRiY2PVr18/SVLnzp01YMAADRs2TAsWLJAkDR8+XMnJyba/OQcAAC5uXh2a5s+fL0nq27evx/gLL7ygIUOGSJLGjh2r48ePKyMjQ+Xl5YqPj1dubq7atGlj1c+ePVu+vr5KS0vT8ePHlZiYqCVLlsjHx8eqWbFihUaOHGl9yy41NVVz585t3AkCAIBmo1k9p8nb8ZwmoOnwnCbg4sVzmgAAAJoRQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAAABsITQAAADYQmgAAAGwgNAEAANhAaAIAALCB0AQAAGADoQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkLTaebNm6eOHTuqZcuWiouL07vvvtvULQEAAC9AaDrFyy+/rMzMTD3++ON6//33deONN2rgwIEqKipq6tYAAEATIzSd4umnn9bQoUP161//Wp07d9acOXMUFRWl+fPnN3VrAACgiRGa/ldNTY3y8vKUlJTkMZ6UlKTNmzc3UVcAAMBb+DZ1A97iyy+/VF1dnSIiIjzGIyIiVFpaesZjqqurVV1dbW273W5JUkVFRaP1WVd9vNHODTRnjfnv7ody9Ou6pm4B8EqN/e/75PlN0zxrHaHpNA6Hw2PbNM16YydNmzZNkydPrjceFRXVKL0B+G7Gs79p6hYANJZpxg/yNkePHpVhfPd7EZr+V1hYmHx8fOqtKpWVldVbfTpp/PjxGjVqlLV94sQJHT58WKGhod8ZtHDxqKioUFRUlIqLixUUFNTU7QC4gPj3/eNimqaOHj0qp9N51jpC0//y9/dXXFyc1q1bpzvvvNMaX7dune64444zHhMQEKCAgACPsbZt2zZmm/BCQUFB/EcVuEjx7/vH42wrTCcRmk4xatQouVwu9ejRQwkJCVq4cKGKior0m9+w7A8AwI8doekUgwcP1ldffaUpU6aopKREMTEx+te//qUOHTo0dWsAAKCJEZpOk5GRoYyMjKZuA81AQECAJk2aVO8SLYDmj3/fOBOHea7v1wEAAICHWwIAANhBaAIAALCB0AQAAGADoQkAAMAGQhPQAPPmzVPHjh3VsmVLxcXF6d13323qlgBcAO+8845SUlLkdDrlcDi0Zs2apm4JXoTQBJynl19+WZmZmXr88cf1/vvv68Ybb9TAgQNVVFTU1K0B+J6qqqrUrVs3zZ07t6lbgRfikQPAeYqPj9fPfvYzzZ8/3xrr3LmzBg0apGnTpjVhZwAuJIfDodWrV2vQoEFN3Qq8BCtNwHmoqalRXl6ekpKSPMaTkpK0efPmJuoKAPBDIDQB5+HLL79UXV2dIiIiPMYjIiJUWlraRF0BAH4IhCagARwOh8e2aZr1xgAAFxdCE3AewsLC5OPjU29VqaysrN7qEwDg4kJoAs6Dv7+/4uLitG7dOo/xdevWqVevXk3UFQDgh+Db1A0Azc2oUaPkcrnUo0cPJSQkaOHChSoqKtJvfvObpm4NwPdUWVmpTz75xNrev3+/8vPzFRISovbt2zdhZ/AGPHIAaIB58+ZpxowZKikpUUxMjGbPnq2bbrqpqdsC8D1t2LBBN998c73xBx98UEuWLPnhG4JXITQBAADYwD1NAAAANhCaAAAAbCA0AQAA2EBoAgAAsIHQBAAAYAOhCQAAwAZCEwAAgA2EJgAXnc8++0wOh0P5+fmSvn1gocPh0JEjR2wd37dvX2VmZn7n/iFDhmjQoEEXtEcA3o/QBOCi16tXL5WUlMgwjKZuxRIVFWU9UV46/2AH4IfH754DcNHz9/dXZGRkU7fhwcfHx+t6AnB2rDQBaLZOnDih6dOn66c//akCAgLUvn17Pfnkk/XqzrSK85///Ed9+vRRq1atFBwcrP79+6u8vPyM75OTkyPDMLR06VKP8cmTJys8PFxBQUF66KGHVFNT43HMDTfcoLZt2yo0NFTJycn69NNPrf2nXp777LPPrN93FhwcLIfDoSFDhnyPTwZAYyA0AWi2xo8fr+nTp2vChAnas2ePVq5cqYiIiHMel5+fr8TERHXt2lVbtmzRpk2blJKSorq6unq12dnZSktL09KlS/XAAw9Y42+99ZYKCwu1fv16vfTSS1q9erUmT55s7a+qqtKoUaO0Y8cOvfXWW2rRooXuvPNOnThxot57REVFadWqVZKkffv2qaSkRM8880xDPhIAjYjLcwCapaNHj+qZZ57R3Llz9eCDD0qSfvKTn+iGG27QZ599dtZjZ8yYoR49emjevHnWWNeuXevVzZs3T7///e/16quvWitBJ/n7++v5559Xq1at1LVrV02ZMkVjxozRH//4R7Vo0UI///nPPeoXL16s8PBw7dmzx7qP6SQfHx+FhIRIksLDw9W2bVu7HwOAHxArTQCapcLCQlVXVysxMfG8jz250nQ2q1atUmZmpnJzc+sFJknq1q2bWrVqZW0nJCSosrJSxcXFkqRPP/1U6enpuvLKKxUUFKSOHTtKkoqKis67XwDegdAEoFkKDAxs1GOvvfZatWvXTi+88IJM07R9bofDIUlKSUnRV199pUWLFmnbtm3atm2bJHnc9wSgeSE0AWiWOnXqpMDAQL311lvnfew111xzzuN+8pOfaP369Xr11Vc1YsSIevt37dql48ePW9tbt27VJZdcossvv1xfffWVCgsL9Yc//EGJiYnq3Lnzd95kfpK/v78knfG+KgDegdAEoFlq2bKlxo0bp7Fjx2rp0qX69NNPtXXrVi1evPicx44fP147duxQRkaGdu/erb1792r+/Pn68ssvPequuuoqrV+/3rpUd6qamhoNHTpUe/bs0RtvvKFJkybpkUceUYsWLRQcHKzQ0FAtXLhQn3zyid5++22NGjXqrD116NBBDodDr732mg4dOqTKysrz/kwANC5CE4Bma8KECcrKytLEiRPVuXNnDR48WGVlZec87qqrrlJubq527dql66+/XgkJCXr11Vfl61v/uzHR0dF6++239dJLLykrK8saT0xMVKdOnXTTTTcpLS1NKSkpeuKJJyRJLVq0UHZ2tvLy8hQTE6NHH31UM2fOPGtPl112mSZPnqzHHntMEREReuSRR87vwwDQ6Bzm+VysBwAA+JFipQkAAMAGQhMAAIANhCYAAAAbCE0AAAA2EJoAAABsIDQBAADYQGgCAACwgdAEAABgA6EJAADABkITAACADYQmAAAAGwhNAAAANvx/EamTkvz66TcAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Let's preprocess the text by lowercasing everything and replacing all numbers with NUM.","metadata":{}},{"cell_type":"code","source":"X_dataset = dataset['headline']\ny_dataset = dataset['clickbait']\n\nprint('Before preprocessing: ')\nfor x in X_dataset[:10]:\n    print(x)\n\n# Lowercase and remove punctuation\nX_dataset = [re.sub(r'[^\\w\\s]', '', x).lower() for x in X_dataset]\n# Replace all numbers with NUM\nX_dataset = [re.sub(r'[0-9]+', 'num', x) for x in X_dataset]\n\n\nprint('\\nAfter preprocessing: ')\nfor x in X_dataset[:10]:\n    print(x)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:45.041025Z","iopub.execute_input":"2023-04-20T07:32:45.042231Z","iopub.status.idle":"2023-04-20T07:32:45.211081Z","shell.execute_reply.started":"2023-04-20T07:32:45.042172Z","shell.execute_reply":"2023-04-20T07:32:45.209690Z"},"trusted":true},"execution_count":80,"outputs":[{"name":"stdout","text":"Before preprocessing: \nShould I Get Bings\nWhich TV Female Friend Group Do You Belong In\nThe New \"Star Wars: The Force Awakens\" Trailer Is Here To Give You Chills\nThis Vine Of New York On \"Celebrity Big Brother\" Is Fucking Perfect\nA Couple Did A Stunning Photo Shoot With Their Baby After Learning She Had An Inoperable Brain Tumor\nHow To Flirt With Queer Girls Without Making A Total Fool Of Yourself\n32 Cute Things To Distract From Your Awkward Thanksgiving\nIf Disney Princesses Were From Florida\nWhat's A Quote Or Lyric That Best Describes Your Depression\nNatalie Dormer And Sam Claflin Play A Game To See How They'd Actually Last In \"The Hunger Games\"\n\nAfter preprocessing: \nshould i get bings\nwhich tv female friend group do you belong in\nthe new star wars the force awakens trailer is here to give you chills\nthis vine of new york on celebrity big brother is fucking perfect\na couple did a stunning photo shoot with their baby after learning she had an inoperable brain tumor\nhow to flirt with queer girls without making a total fool of yourself\nnum cute things to distract from your awkward thanksgiving\nif disney princesses were from florida\nwhats a quote or lyric that best describes your depression\nnatalie dormer and sam claflin play a game to see how theyd actually last in the hunger games\n","output_type":"stream"}]},{"cell_type":"code","source":"# Split into train and test data\nX_train_unfeaturized, X_test_unfeaturized, y_train, y_test = train_test_split(X_dataset, y_dataset, test_size=0.2, random_state=1234)\nprint(X_train_unfeaturized[:5])\nprint(y_train[:5])","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:45.212653Z","iopub.execute_input":"2023-04-20T07:32:45.213141Z","iopub.status.idle":"2023-04-20T07:32:45.235568Z","shell.execute_reply.started":"2023-04-20T07:32:45.213091Z","shell.execute_reply":"2023-04-20T07:32:45.234054Z"},"trusted":true},"execution_count":81,"outputs":[{"name":"stdout","text":"['which marnie from halloweentown are you', 'aig sells its japanese headquarters for num billion', 'if disney princesses lived in canada', 'nasa schedules launch date for cometchasing probe', 'num times we were all princess eleanor from the royals']\n13536    1\n31326    0\n14722    1\n24503    0\n9850     1\nName: clickbait, dtype: int64\n","output_type":"stream"}]},{"cell_type":"markdown","source":"The training text contains 21,755 unique tokens, but we will focus on the 20,000 most frequent since infrequent words may be things like celebrity names or brands. The longest headline in the training data is 26 words long, so we will set the maximum length of a headline to be 30.","metadata":{}},{"cell_type":"code","source":"# Count the unique words\nall_train_text = ' '.join(X_train_unfeaturized)\nall_train_words = all_train_text.split(' ')\nunique_tokens = set(all_train_words)\nvocab_size = len(unique_tokens)\nprint('Unique token count: ' + str(vocab_size))\n\n# Find lengths of headlines \nheadline_lengths = [len(x.split()) for x in X_train_unfeaturized]\nmax_headline_length = np.array(headline_lengths).max()\nprint('Most words in a headline: ' + str(max_headline_length))\n\n\nvocab_size = 20000\nmax_length = 30","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:45.237070Z","iopub.execute_input":"2023-04-20T07:32:45.237437Z","iopub.status.idle":"2023-04-20T07:32:45.343070Z","shell.execute_reply.started":"2023-04-20T07:32:45.237402Z","shell.execute_reply":"2023-04-20T07:32:45.342066Z"},"trusted":true},"execution_count":82,"outputs":[{"name":"stdout","text":"Unique token count: 21755\nMost words in a headline: 26\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Sequential Neural Nets\nFor these basic sequential neural nets we will represent the website text as a bag of words using their tf-idf scores.","metadata":{}},{"cell_type":"code","source":"# Vectorize the text data using the top 20,000 words instead of all the words\ntokenizer = Tokenizer(num_words=vocab_size)\ntokenizer.fit_on_texts(X_train_unfeaturized)\n\nX_train = tokenizer.texts_to_matrix(X_train_unfeaturized, mode='tfidf')\nX_test = tokenizer.texts_to_matrix(X_test_unfeaturized, mode='tfidf')\n\n# Display vectorized features\nprint('Feature vectors:')\nprint(X_train[:5])","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:45.344169Z","iopub.execute_input":"2023-04-20T07:32:45.344519Z","iopub.status.idle":"2023-04-20T07:32:48.714315Z","shell.execute_reply.started":"2023-04-20T07:32:45.344486Z","shell.execute_reply":"2023-04-20T07:32:48.712905Z"},"trusted":true},"execution_count":83,"outputs":[{"name":"stdout","text":"Feature vectors:\n[[0.         0.         0.         ... 0.         0.         0.        ]\n [0.         1.47633231 0.         ... 0.         0.         0.        ]\n [0.         0.         0.         ... 0.         0.         0.        ]\n [0.         0.         0.         ... 0.         0.         0.        ]\n [0.         1.47633231 0.         ... 0.         0.         0.        ]]\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Attempt 3\nThis model uses a single hidden layer of 16 nodes with relu activation and an output layer that uses sigmoid activation.","metadata":{}},{"cell_type":"code","source":"# Build a sequential neural net\nmodel = models.Sequential()\nmodel.add(layers.Dense(16, input_dim=vocab_size,activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n \nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n \nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:32:48.715895Z","iopub.execute_input":"2023-04-20T07:32:48.716400Z","iopub.status.idle":"2023-04-20T07:33:29.962800Z","shell.execute_reply.started":"2023-04-20T07:32:48.716351Z","shell.execute_reply":"2023-04-20T07:33:29.961396Z"},"trusted":true},"execution_count":84,"outputs":[{"name":"stdout","text":"Model: \"sequential_30\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n dense_43 (Dense)            (None, 16)                320016    \n                                                                 \n dense_44 (Dense)            (None, 1)                 17        \n                                                                 \n=================================================================\nTotal params: 320,033\nTrainable params: 320,033\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 3s 17ms/step - loss: 0.2162 - accuracy: 0.9455 - val_loss: 0.0799 - val_accuracy: 0.9758\nEpoch 2/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0496 - accuracy: 0.9841 - val_loss: 0.0613 - val_accuracy: 0.9836\nEpoch 3/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0298 - accuracy: 0.9903 - val_loss: 0.0585 - val_accuracy: 0.9832\nEpoch 4/15\n180/180 [==============================] - 2s 13ms/step - loss: 0.0201 - accuracy: 0.9937 - val_loss: 0.0592 - val_accuracy: 0.9840\nEpoch 5/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0143 - accuracy: 0.9951 - val_loss: 0.0618 - val_accuracy: 0.9836\nEpoch 6/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0103 - accuracy: 0.9966 - val_loss: 0.0650 - val_accuracy: 0.9824\nEpoch 7/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0076 - accuracy: 0.9975 - val_loss: 0.0679 - val_accuracy: 0.9812\nEpoch 8/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0056 - accuracy: 0.9984 - val_loss: 0.0682 - val_accuracy: 0.9812\nEpoch 9/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0043 - accuracy: 0.9988 - val_loss: 0.0724 - val_accuracy: 0.9801\nEpoch 10/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0033 - accuracy: 0.9993 - val_loss: 0.0750 - val_accuracy: 0.9793\nEpoch 11/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0026 - accuracy: 0.9993 - val_loss: 0.0767 - val_accuracy: 0.9797\nEpoch 12/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0021 - accuracy: 0.9994 - val_loss: 0.0790 - val_accuracy: 0.9789\nEpoch 13/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0016 - accuracy: 0.9995 - val_loss: 0.0823 - val_accuracy: 0.9793\nEpoch 14/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0013 - accuracy: 0.9996 - val_loss: 0.0846 - val_accuracy: 0.9785\nEpoch 15/15\n180/180 [==============================] - 2s 12ms/step - loss: 0.0010 - accuracy: 0.9997 - val_loss: 0.0868 - val_accuracy: 0.9785\n200/200 [==============================] - 1s 2ms/step\n[[3108   91]\n [  71 3130]]\n              precision    recall  f1-score   support\n\n           0       0.98      0.97      0.97      3199\n           1       0.97      0.98      0.97      3201\n\n    accuracy                           0.97      6400\n   macro avg       0.97      0.97      0.97      6400\nweighted avg       0.97      0.97      0.97      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Attempt 2\nThis sequential neural net adds an embedding layer to reduce sparsity, but accuracy is drastically reduced.","metadata":{}},{"cell_type":"code","source":"# Build a sequential neural net\nmodel = models.Sequential()\nmodel.add(layers.Embedding(input_dim=vocab_size, output_dim=8, input_length=vocab_size))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(16, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n \nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:33:29.965525Z","iopub.execute_input":"2023-04-20T07:33:29.966647Z","iopub.status.idle":"2023-04-20T07:43:00.946767Z","shell.execute_reply.started":"2023-04-20T07:33:29.966552Z","shell.execute_reply":"2023-04-20T07:43:00.944575Z"},"trusted":true},"execution_count":85,"outputs":[{"name":"stdout","text":"Model: \"sequential_31\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_15 (Embedding)    (None, 20000, 8)          160000    \n                                                                 \n flatten_8 (Flatten)         (None, 160000)            0         \n                                                                 \n dense_45 (Dense)            (None, 16)                2560016   \n                                                                 \n dense_46 (Dense)            (None, 1)                 17        \n                                                                 \n=================================================================\nTotal params: 2,720,033\nTrainable params: 2,720,033\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 37s 201ms/step - loss: 0.6933 - accuracy: 0.5007 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 2/15\n180/180 [==============================] - 36s 200ms/step - loss: 0.6932 - accuracy: 0.5003 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 3/15\n180/180 [==============================] - 36s 198ms/step - loss: 0.6932 - accuracy: 0.5001 - val_loss: 0.6931 - val_accuracy: 0.4984\nEpoch 4/15\n180/180 [==============================] - 35s 195ms/step - loss: 0.6932 - accuracy: 0.4953 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 5/15\n180/180 [==============================] - 36s 197ms/step - loss: 0.6932 - accuracy: 0.4956 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 6/15\n180/180 [==============================] - 35s 197ms/step - loss: 0.6932 - accuracy: 0.4988 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 7/15\n180/180 [==============================] - 35s 196ms/step - loss: 0.6932 - accuracy: 0.4939 - val_loss: 0.6931 - val_accuracy: 0.4984\nEpoch 8/15\n180/180 [==============================] - 55s 305ms/step - loss: 0.6932 - accuracy: 0.4940 - val_loss: 0.6931 - val_accuracy: 0.4984\nEpoch 9/15\n180/180 [==============================] - 37s 204ms/step - loss: 0.6932 - accuracy: 0.4999 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 10/15\n180/180 [==============================] - 35s 197ms/step - loss: 0.6932 - accuracy: 0.4983 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 11/15\n180/180 [==============================] - 36s 199ms/step - loss: 0.6932 - accuracy: 0.4970 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 12/15\n180/180 [==============================] - 35s 197ms/step - loss: 0.6932 - accuracy: 0.4986 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 13/15\n180/180 [==============================] - 36s 201ms/step - loss: 0.6932 - accuracy: 0.4950 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 14/15\n180/180 [==============================] - 36s 201ms/step - loss: 0.6932 - accuracy: 0.4990 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 15/15\n180/180 [==============================] - 36s 201ms/step - loss: 0.6932 - accuracy: 0.4997 - val_loss: 0.6931 - val_accuracy: 0.4984\n200/200 [==============================] - 3s 13ms/step\n[[   0 3199]\n [   0 3201]]\n              precision    recall  f1-score   support\n\n           0       0.00      0.00      0.00      3199\n           1       0.50      1.00      0.67      3201\n\n    accuracy                           0.50      6400\n   macro avg       0.25      0.50      0.33      6400\nweighted avg       0.25      0.50      0.33      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Convolutional Neural Nets\nFor a CNN, order of the words matters since we are using a sliding window approach to the data. To preserve the ordering information we will use a TextVectorization layer as an encoder to map words to integers. Each observation will have its words converted to a list of integers representing a word with 0 padding the rest of the array for examples that have fewer words than the longest observed.","metadata":{}},{"cell_type":"code","source":"X_train = X_train_unfeaturized\nX_test = X_test_unfeaturized\n\nprint('Unencoded observation example:')\nprint(X_train[0])\n\n# Create the encoder layer\nfeature_encoder = layers.TextVectorization(max_tokens=vocab_size,  \n                                                    output_sequence_length=max_length)\nfeature_encoder.adapt(X_train)\n\nprint('\\nEncoded vocabulary example:')\nvocab = np.array(feature_encoder.get_vocabulary())\nprint(vocab[:20])\n\nencoded_example = feature_encoder(X_train)[0].numpy()\nprint('\\nEncoded example:')\nprint(encoded_example)\n\nprint('\\nConverting the first word of example back: ')\nprint(str(encoded_example[0]) + ': ' + vocab[encoded_example[0]])","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:43:00.962231Z","iopub.execute_input":"2023-04-20T07:43:00.963565Z","iopub.status.idle":"2023-04-20T07:43:02.734098Z","shell.execute_reply.started":"2023-04-20T07:43:00.963518Z","shell.execute_reply":"2023-04-20T07:43:02.732820Z"},"trusted":true},"execution_count":86,"outputs":[{"name":"stdout","text":"Unencoded observation example:\nwhich marnie from halloweentown are you\n\nEncoded vocabulary example:\n['' '[UNK]' 'num' 'to' 'in' 'the' 'of' 'you' 'a' 'for' 'and' 'on' 'your'\n 'are' 'is' 'that' 'this' 'with' 'at' 'will']\n\nEncoded example:\n[  29 5975   21 3318   13    7    0    0    0    0    0    0    0    0\n    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n    0    0]\n\nConverting the first word of example back: \n29: which\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Attempt 1\nThis CNN model uses 1 convolutional layer and an embedding layer of size 8.","metadata":{}},{"cell_type":"code","source":"X_train = feature_encoder(X_train_unfeaturized)\nX_test = feature_encoder(X_test_unfeaturized)\n\n# Build a CNN\nmodel = models.Sequential()\nmodel.add(layers.Embedding(vocab_size, 8, input_length=max_length)) \nmodel.add(layers.Conv1D(32, 7, activation='relu')) \nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(layers.Dense(1, activation = 'sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:43:02.737131Z","iopub.execute_input":"2023-04-20T07:43:02.738353Z","iopub.status.idle":"2023-04-20T07:43:24.915606Z","shell.execute_reply.started":"2023-04-20T07:43:02.738313Z","shell.execute_reply":"2023-04-20T07:43:24.913834Z"},"trusted":true},"execution_count":87,"outputs":[{"name":"stdout","text":"Model: \"sequential_32\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_16 (Embedding)    (None, 30, 8)             160000    \n                                                                 \n conv1d_5 (Conv1D)           (None, 24, 32)            1824      \n                                                                 \n global_max_pooling1d_3 (Glo  (None, 32)               0         \n balMaxPooling1D)                                                \n                                                                 \n dense_47 (Dense)            (None, 1)                 33        \n                                                                 \n=================================================================\nTotal params: 161,857\nTrainable params: 161,857\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 2s 8ms/step - loss: 0.4877 - accuracy: 0.8368 - val_loss: 0.2124 - val_accuracy: 0.9469\nEpoch 2/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.1306 - accuracy: 0.9582 - val_loss: 0.1067 - val_accuracy: 0.9637\nEpoch 3/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0781 - accuracy: 0.9724 - val_loss: 0.0889 - val_accuracy: 0.9691\nEpoch 4/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0595 - accuracy: 0.9797 - val_loss: 0.0857 - val_accuracy: 0.9688\nEpoch 5/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0483 - accuracy: 0.9832 - val_loss: 0.0786 - val_accuracy: 0.9707\nEpoch 6/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0402 - accuracy: 0.9864 - val_loss: 0.0771 - val_accuracy: 0.9742\nEpoch 7/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0342 - accuracy: 0.9882 - val_loss: 0.0754 - val_accuracy: 0.9734\nEpoch 8/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0294 - accuracy: 0.9899 - val_loss: 0.0760 - val_accuracy: 0.9734\nEpoch 9/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0254 - accuracy: 0.9913 - val_loss: 0.0771 - val_accuracy: 0.9738\nEpoch 10/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0221 - accuracy: 0.9925 - val_loss: 0.0773 - val_accuracy: 0.9742\nEpoch 11/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0193 - accuracy: 0.9932 - val_loss: 0.0775 - val_accuracy: 0.9746\nEpoch 12/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0169 - accuracy: 0.9948 - val_loss: 0.0809 - val_accuracy: 0.9750\nEpoch 13/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.9955 - val_loss: 0.0790 - val_accuracy: 0.9742\nEpoch 14/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0130 - accuracy: 0.9964 - val_loss: 0.0811 - val_accuracy: 0.9742\nEpoch 15/15\n180/180 [==============================] - 1s 6ms/step - loss: 0.0114 - accuracy: 0.9968 - val_loss: 0.0818 - val_accuracy: 0.9742\n200/200 [==============================] - 0s 2ms/step\n[[3118   81]\n [  78 3123]]\n              precision    recall  f1-score   support\n\n           0       0.98      0.97      0.98      3199\n           1       0.97      0.98      0.98      3201\n\n    accuracy                           0.98      6400\n   macro avg       0.98      0.98      0.98      6400\nweighted avg       0.98      0.98      0.98      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Attempt 2\nThis CNN model uses 1 convolutional layer and an embedding layer of size 16. The 4th root of the vocab size 20,000 is 11 which is between 8 and 16 (powers of 2 for best performance). However, increasing the embedding layer to the next size up doesn't improve accuracy.","metadata":{}},{"cell_type":"code","source":"X_train = feature_encoder(X_train_unfeaturized)\nX_test = feature_encoder(X_test_unfeaturized)\n\n# Build a CNN\nmodel = models.Sequential()\nmodel.add(layers.Embedding(vocab_size, 16, input_length=max_length)) \nmodel.add(layers.Conv1D(32, 7, activation='relu')) \nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(layers.Dense(1, activation = 'sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:43:24.917200Z","iopub.execute_input":"2023-04-20T07:43:24.917540Z","iopub.status.idle":"2023-04-20T07:43:47.270773Z","shell.execute_reply.started":"2023-04-20T07:43:24.917508Z","shell.execute_reply":"2023-04-20T07:43:47.269654Z"},"trusted":true},"execution_count":88,"outputs":[{"name":"stdout","text":"Model: \"sequential_33\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_17 (Embedding)    (None, 30, 16)            320000    \n                                                                 \n conv1d_6 (Conv1D)           (None, 24, 32)            3616      \n                                                                 \n global_max_pooling1d_4 (Glo  (None, 32)               0         \n balMaxPooling1D)                                                \n                                                                 \n dense_48 (Dense)            (None, 1)                 33        \n                                                                 \n=================================================================\nTotal params: 323,649\nTrainable params: 323,649\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 2s 8ms/step - loss: 0.4234 - accuracy: 0.8706 - val_loss: 0.1613 - val_accuracy: 0.9516\nEpoch 2/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.1094 - accuracy: 0.9625 - val_loss: 0.0991 - val_accuracy: 0.9637\nEpoch 3/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0695 - accuracy: 0.9764 - val_loss: 0.0854 - val_accuracy: 0.9695\nEpoch 4/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0523 - accuracy: 0.9823 - val_loss: 0.0780 - val_accuracy: 0.9711\nEpoch 5/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0416 - accuracy: 0.9859 - val_loss: 0.0755 - val_accuracy: 0.9715\nEpoch 6/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0338 - accuracy: 0.9883 - val_loss: 0.0760 - val_accuracy: 0.9734\nEpoch 7/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0278 - accuracy: 0.9907 - val_loss: 0.0754 - val_accuracy: 0.9750\nEpoch 8/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0228 - accuracy: 0.9923 - val_loss: 0.0761 - val_accuracy: 0.9750\nEpoch 9/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0188 - accuracy: 0.9942 - val_loss: 0.0788 - val_accuracy: 0.9742\nEpoch 10/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0158 - accuracy: 0.9951 - val_loss: 0.0774 - val_accuracy: 0.9758\nEpoch 11/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.9962 - val_loss: 0.0780 - val_accuracy: 0.9754\nEpoch 12/15\n180/180 [==============================] - 1s 8ms/step - loss: 0.0108 - accuracy: 0.9968 - val_loss: 0.0797 - val_accuracy: 0.9746\nEpoch 13/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.9975 - val_loss: 0.0810 - val_accuracy: 0.9754\nEpoch 14/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0076 - accuracy: 0.9978 - val_loss: 0.0815 - val_accuracy: 0.9750\nEpoch 15/15\n180/180 [==============================] - 1s 7ms/step - loss: 0.0061 - accuracy: 0.9986 - val_loss: 0.0852 - val_accuracy: 0.9734\n200/200 [==============================] - 0s 2ms/step\n[[3104   95]\n [  74 3127]]\n              precision    recall  f1-score   support\n\n           0       0.98      0.97      0.97      3199\n           1       0.97      0.98      0.97      3201\n\n    accuracy                           0.97      6400\n   macro avg       0.97      0.97      0.97      6400\nweighted avg       0.97      0.97      0.97      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Recurrent Neural Nets\nLike with CNNs, order of the words matters so we will use the same encoding scheme.","metadata":{}},{"cell_type":"code","source":"X_train = X_train_unfeaturized\nX_test = X_test_unfeaturized\n\n# Create the encoder layer\nfeature_encoder = layers.TextVectorization(max_tokens=vocab_size,  \n                                                    output_sequence_length=max_length)\nfeature_encoder.adapt(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:43:47.272058Z","iopub.execute_input":"2023-04-20T07:43:47.272413Z","iopub.status.idle":"2023-04-20T07:43:48.653874Z","shell.execute_reply.started":"2023-04-20T07:43:47.272379Z","shell.execute_reply":"2023-04-20T07:43:48.652374Z"},"trusted":true},"execution_count":89,"outputs":[]},{"cell_type":"markdown","source":"### Attempt 1\nThis model uses an embedding layer and 1 simple hidden RNN layer.","metadata":{}},{"cell_type":"code","source":"X_train = feature_encoder(X_train_unfeaturized)\nX_test = feature_encoder(X_test_unfeaturized)\n\n# Build an RNN\nmodel = models.Sequential()\nmodel.add(layers.Embedding(vocab_size, 16, input_length=max_length))\nmodel.add(layers.SimpleRNN(32))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:43:48.655765Z","iopub.execute_input":"2023-04-20T07:43:48.656160Z","iopub.status.idle":"2023-04-20T07:44:19.295285Z","shell.execute_reply.started":"2023-04-20T07:43:48.656124Z","shell.execute_reply":"2023-04-20T07:44:19.290497Z"},"trusted":true},"execution_count":90,"outputs":[{"name":"stdout","text":"Model: \"sequential_34\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_18 (Embedding)    (None, 30, 16)            320000    \n                                                                 \n simple_rnn_1 (SimpleRNN)    (None, 32)                1568      \n                                                                 \n dense_49 (Dense)            (None, 1)                 33        \n                                                                 \n=================================================================\nTotal params: 321,601\nTrainable params: 321,601\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 3s 12ms/step - loss: 0.3350 - accuracy: 0.8644 - val_loss: 0.2656 - val_accuracy: 0.8980\nEpoch 2/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.1662 - accuracy: 0.9411 - val_loss: 0.1955 - val_accuracy: 0.9320\nEpoch 3/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.1057 - accuracy: 0.9630 - val_loss: 0.2216 - val_accuracy: 0.9223\nEpoch 4/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0743 - accuracy: 0.9746 - val_loss: 0.2125 - val_accuracy: 0.9285\nEpoch 5/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0548 - accuracy: 0.9816 - val_loss: 0.2109 - val_accuracy: 0.9293\nEpoch 6/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0411 - accuracy: 0.9863 - val_loss: 0.2403 - val_accuracy: 0.9184\nEpoch 7/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0307 - accuracy: 0.9896 - val_loss: 0.2597 - val_accuracy: 0.9293\nEpoch 8/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0238 - accuracy: 0.9929 - val_loss: 0.2366 - val_accuracy: 0.9297\nEpoch 9/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0173 - accuracy: 0.9948 - val_loss: 0.2629 - val_accuracy: 0.9262\nEpoch 10/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0155 - accuracy: 0.9958 - val_loss: 0.2798 - val_accuracy: 0.9270\nEpoch 11/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0117 - accuracy: 0.9967 - val_loss: 0.3267 - val_accuracy: 0.9145\nEpoch 12/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0108 - accuracy: 0.9970 - val_loss: 0.2955 - val_accuracy: 0.9207\nEpoch 13/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0082 - accuracy: 0.9975 - val_loss: 0.2751 - val_accuracy: 0.9191\nEpoch 14/15\n180/180 [==============================] - 2s 11ms/step - loss: 0.0075 - accuracy: 0.9977 - val_loss: 0.2939 - val_accuracy: 0.9277\nEpoch 15/15\n180/180 [==============================] - 2s 10ms/step - loss: 0.0063 - accuracy: 0.9982 - val_loss: 0.3373 - val_accuracy: 0.9227\n200/200 [==============================] - 1s 3ms/step\n[[2952  247]\n [ 201 3000]]\n              precision    recall  f1-score   support\n\n           0       0.94      0.92      0.93      3199\n           1       0.92      0.94      0.93      3201\n\n    accuracy                           0.93      6400\n   macro avg       0.93      0.93      0.93      6400\nweighted avg       0.93      0.93      0.93      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Attempt 2\nThis model uses a more powerful LSTM but gets similar results to the simple RNN","metadata":{}},{"cell_type":"code","source":"X_train = feature_encoder(X_train_unfeaturized)\nX_test = feature_encoder(X_test_unfeaturized)\n\n# Build an LSTM\nmodel = models.Sequential()\nmodel.add(layers.Embedding(vocab_size, 16))\nmodel.add(layers.LSTM(32))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:44:19.297264Z","iopub.execute_input":"2023-04-20T07:44:19.298359Z","iopub.status.idle":"2023-04-20T07:45:45.212295Z","shell.execute_reply.started":"2023-04-20T07:44:19.298313Z","shell.execute_reply":"2023-04-20T07:45:45.210923Z"},"trusted":true},"execution_count":91,"outputs":[{"name":"stdout","text":"Model: \"sequential_35\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_19 (Embedding)    (None, None, 16)          320000    \n                                                                 \n lstm_1 (LSTM)               (None, 32)                6272      \n                                                                 \n dense_50 (Dense)            (None, 1)                 33        \n                                                                 \n=================================================================\nTotal params: 326,305\nTrainable params: 326,305\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 6s 23ms/step - loss: 0.5827 - accuracy: 0.6286 - val_loss: 0.2029 - val_accuracy: 0.9406\nEpoch 2/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.1262 - accuracy: 0.9602 - val_loss: 0.0851 - val_accuracy: 0.9719\nEpoch 3/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0595 - accuracy: 0.9799 - val_loss: 0.0830 - val_accuracy: 0.9680\nEpoch 4/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0397 - accuracy: 0.9877 - val_loss: 0.0748 - val_accuracy: 0.9742\nEpoch 5/15\n180/180 [==============================] - 4s 22ms/step - loss: 0.0288 - accuracy: 0.9911 - val_loss: 0.0665 - val_accuracy: 0.9770\nEpoch 6/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0228 - accuracy: 0.9932 - val_loss: 0.0703 - val_accuracy: 0.9766\nEpoch 7/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0160 - accuracy: 0.9955 - val_loss: 0.0761 - val_accuracy: 0.9754\nEpoch 8/15\n180/180 [==============================] - 4s 21ms/step - loss: 0.0120 - accuracy: 0.9968 - val_loss: 0.1038 - val_accuracy: 0.9680\nEpoch 9/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0100 - accuracy: 0.9974 - val_loss: 0.0981 - val_accuracy: 0.9730\nEpoch 10/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0079 - accuracy: 0.9979 - val_loss: 0.0834 - val_accuracy: 0.9773\nEpoch 11/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0056 - accuracy: 0.9986 - val_loss: 0.1477 - val_accuracy: 0.9703\nEpoch 12/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0041 - accuracy: 0.9990 - val_loss: 0.1567 - val_accuracy: 0.9730\nEpoch 13/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0036 - accuracy: 0.9990 - val_loss: 0.1432 - val_accuracy: 0.9734\nEpoch 14/15\n180/180 [==============================] - 4s 22ms/step - loss: 0.0027 - accuracy: 0.9994 - val_loss: 0.1962 - val_accuracy: 0.9699\nEpoch 15/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.0026 - accuracy: 0.9993 - val_loss: 0.1525 - val_accuracy: 0.9684\n200/200 [==============================] - 2s 6ms/step\n[[3039  160]\n [  48 3153]]\n              precision    recall  f1-score   support\n\n           0       0.98      0.95      0.97      3199\n           1       0.95      0.99      0.97      3201\n\n    accuracy                           0.97      6400\n   macro avg       0.97      0.97      0.97      6400\nweighted avg       0.97      0.97      0.97      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Attempt 3\nThis model uses a gated recurrent unit. The accuracy is surprisingly worse than the LSTM.","metadata":{}},{"cell_type":"code","source":"X_train = feature_encoder(X_train_unfeaturized)\nX_test = feature_encoder(X_test_unfeaturized)\n\n# Build a CNN\nmodel = models.Sequential()\nmodel.add(layers.Embedding(vocab_size, 16, input_length=max_length))\nmodel.add(layers.GRU(32))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(X_train, y_train,\n                    batch_size=128,\n                    epochs=15,\n                    verbose=1,\n                    validation_split=0.1)\n\n# Make predictions and check performance metrics\npred = model.predict(X_test)\npred_labels = [1 if p>= 0.5 else 0 for p in pred]\nprint(confusion_matrix(y_test, pred_labels))\nprint(classification_report(y_test, pred_labels, zero_division=0))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T07:45:45.214418Z","iopub.execute_input":"2023-04-20T07:45:45.214779Z","iopub.status.idle":"2023-04-20T07:46:38.957767Z","shell.execute_reply.started":"2023-04-20T07:45:45.214745Z","shell.execute_reply":"2023-04-20T07:46:38.956480Z"},"trusted":true},"execution_count":92,"outputs":[{"name":"stdout","text":"Model: \"sequential_36\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n embedding_20 (Embedding)    (None, 30, 16)            320000    \n                                                                 \n gru_1 (GRU)                 (None, 32)                4800      \n                                                                 \n dense_51 (Dense)            (None, 1)                 33        \n                                                                 \n=================================================================\nTotal params: 324,833\nTrainable params: 324,833\nNon-trainable params: 0\n_________________________________________________________________\nNone\nEpoch 1/15\n180/180 [==============================] - 6s 21ms/step - loss: 0.6934 - accuracy: 0.4977 - val_loss: 0.6933 - val_accuracy: 0.4984\nEpoch 2/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6934 - accuracy: 0.4957 - val_loss: 0.6932 - val_accuracy: 0.5016\nEpoch 3/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6932 - accuracy: 0.4984 - val_loss: 0.6937 - val_accuracy: 0.4984\nEpoch 4/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6933 - accuracy: 0.4964 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 5/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6932 - accuracy: 0.4987 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 6/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6933 - accuracy: 0.4990 - val_loss: 0.6934 - val_accuracy: 0.4984\nEpoch 7/15\n180/180 [==============================] - 3s 19ms/step - loss: 0.6933 - accuracy: 0.4974 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 8/15\n180/180 [==============================] - 4s 20ms/step - loss: 0.6933 - accuracy: 0.4980 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 9/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6933 - accuracy: 0.4950 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 10/15\n180/180 [==============================] - 3s 19ms/step - loss: 0.6933 - accuracy: 0.4948 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 11/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6932 - accuracy: 0.4940 - val_loss: 0.6932 - val_accuracy: 0.4984\nEpoch 12/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6932 - accuracy: 0.5001 - val_loss: 0.6932 - val_accuracy: 0.5016\nEpoch 13/15\n180/180 [==============================] - 3s 18ms/step - loss: 0.6932 - accuracy: 0.5006 - val_loss: 0.6932 - val_accuracy: 0.5016\nEpoch 14/15\n180/180 [==============================] - 3s 17ms/step - loss: 0.6932 - accuracy: 0.4960 - val_loss: 0.6931 - val_accuracy: 0.5016\nEpoch 15/15\n180/180 [==============================] - 3s 17ms/step - loss: 0.6932 - accuracy: 0.5024 - val_loss: 0.6931 - val_accuracy: 0.5016\n200/200 [==============================] - 1s 5ms/step\n[[3199    0]\n [3201    0]]\n              precision    recall  f1-score   support\n\n           0       0.50      1.00      0.67      3199\n           1       0.00      0.00      0.00      3201\n\n    accuracy                           0.50      6400\n   macro avg       0.25      0.50      0.33      6400\nweighted avg       0.25      0.50      0.33      6400\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Summary\nAll models performed well on this simple task except for the sequential neural network with an embedding layer and gated recurrent unit. For the sequential neural net this may be because the embedding used bag of words instead of maintaining data about the order of words. ","metadata":{}}]}