ntbook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ef558f64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['UID', 'ph_no', 'cvv', 'credit_card_number', 'job', 'email', 'url',\n",
       "       'country', 'emoji', 'name',\n",
       "       ...\n",
       "       'qhUzJ', 'FpCOT', 'zEnW3', 'ASDn5', 'vF2is', 'pZijn', 'WUc3c', 'sCIyG',\n",
       "       'qaERi', 'state'],\n",
       "      dtype='object', length=111)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "#due to the whole processing being time intensive it was done beforehand and saved in\n",
    "#train_processed3.csv file\n",
    "if processed := os.path.isfile(\"data/train_processed.csv\"):\n",
    "    data = pd.read_csv(\"data/train_processed.csv\")\n",
    "else:\n",
    "    data = pd.read_csv(\"data/train.csv\")\n",
    "    \n",
    "rows = data.shape[0]\n",
    "cols = data.shape[1]\n",
    "collist = data.columns\n",
    "collist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06e5ab0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>ph_no</th>\n",
       "      <th>cvv</th>\n",
       "      <th>credit_card_number</th>\n",
       "      <th>job</th>\n",
       "      <th>email</th>\n",
       "      <th>url</th>\n",
       "      <th>country</th>\n",
       "      <th>emoji</th>\n",
       "      <th>name</th>\n",
       "      <th>...</th>\n",
       "      <th>qhUzJ</th>\n",
       "      <th>FpCOT</th>\n",
       "      <th>zEnW3</th>\n",
       "      <th>ASDn5</th>\n",
       "      <th>vF2is</th>\n",
       "      <th>pZijn</th>\n",
       "      <th>WUc3c</th>\n",
       "      <th>sCIyG</th>\n",
       "      <th>qaERi</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>T77962103</td>\n",
       "      <td>7059868985</td>\n",
       "      <td>673.0</td>\n",
       "      <td>2.131868e+14</td>\n",
       "      <td>Copy</td>\n",
       "      <td>guerrerodavid@example.org</td>\n",
       "      <td>http://garcia.org/</td>\n",
       "      <td>Gabon</td>\n",
       "      <td>📑</td>\n",
       "      <td>Cathy Cherry</td>\n",
       "      <td>...</td>\n",
       "      <td>9.028569</td>\n",
       "      <td>-0.173742</td>\n",
       "      <td>-16.438254</td>\n",
       "      <td>-5.683709</td>\n",
       "      <td>4.495021</td>\n",
       "      <td>1.397634</td>\n",
       "      <td>-2.161601</td>\n",
       "      <td>7.619064</td>\n",
       "      <td>-7.002747</td>\n",
       "      <td>Despair</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>O92591533</td>\n",
       "      <td>+1-288-810-2425x013</td>\n",
       "      <td>983.0</td>\n",
       "      <td>4.820055e+15</td>\n",
       "      <td>Interior and spatial designer</td>\n",
       "      <td>michaela47@example.org</td>\n",
       "      <td>http://ortiz.com/</td>\n",
       "      <td>Bermuda</td>\n",
       "      <td>🥷🏿</td>\n",
       "      <td>Austin Graham</td>\n",
       "      <td>...</td>\n",
       "      <td>-40.773924</td>\n",
       "      <td>3.836528</td>\n",
       "      <td>-20.503862</td>\n",
       "      <td>4.345120</td>\n",
       "      <td>-0.199043</td>\n",
       "      <td>3.414072</td>\n",
       "      <td>8.673901</td>\n",
       "      <td>-0.635993</td>\n",
       "      <td>-11.039355</td>\n",
       "      <td>Worthlessness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>O31883571</td>\n",
       "      <td>791.502.4387x7276</td>\n",
       "      <td>855.0</td>\n",
       "      <td>4.031429e+15</td>\n",
       "      <td>Scientist, water quality</td>\n",
       "      <td>zhoffman@example.org</td>\n",
       "      <td>http://kelly.com/</td>\n",
       "      <td>Sao Tome and Principe</td>\n",
       "      <td>👩‍💻</td>\n",
       "      <td>Frank Smith</td>\n",
       "      <td>...</td>\n",
       "      <td>32.831181</td>\n",
       "      <td>6.673810</td>\n",
       "      <td>31.545004</td>\n",
       "      <td>-1.118935</td>\n",
       "      <td>-2.109316</td>\n",
       "      <td>6.773527</td>\n",
       "      <td>1.718408</td>\n",
       "      <td>-35.955251</td>\n",
       "      <td>8.675279</td>\n",
       "      <td>Stress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M18080565</td>\n",
       "      <td>2522308761</td>\n",
       "      <td>806.0</td>\n",
       "      <td>3.036423e+13</td>\n",
       "      <td>Toxicologist</td>\n",
       "      <td>hprice@example.net</td>\n",
       "      <td>http://mann.info/</td>\n",
       "      <td>Korea</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Austin Young</td>\n",
       "      <td>...</td>\n",
       "      <td>-11.829455</td>\n",
       "      <td>-10.870125</td>\n",
       "      <td>26.131133</td>\n",
       "      <td>2.115912</td>\n",
       "      <td>7.373067</td>\n",
       "      <td>-0.238810</td>\n",
       "      <td>-5.427878</td>\n",
       "      <td>24.501781</td>\n",
       "      <td>-0.845627</td>\n",
       "      <td>Fatigue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q74073854</td>\n",
       "      <td>683-521-2001x423</td>\n",
       "      <td>424.0</td>\n",
       "      <td>4.610207e+15</td>\n",
       "      <td>Software engineer</td>\n",
       "      <td>michael61@example.net</td>\n",
       "      <td>https://www.mitchell.biz/</td>\n",
       "      <td>Russian Federation</td>\n",
       "      <td>💆🏿‍♂️</td>\n",
       "      <td>Kimberly Wilson</td>\n",
       "      <td>...</td>\n",
       "      <td>10.957430</td>\n",
       "      <td>-1.530905</td>\n",
       "      <td>-17.670061</td>\n",
       "      <td>0.132419</td>\n",
       "      <td>3.696047</td>\n",
       "      <td>4.588960</td>\n",
       "      <td>-3.735829</td>\n",
       "      <td>18.194147</td>\n",
       "      <td>10.404818</td>\n",
       "      <td>Tears</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139995</th>\n",
       "      <td>363253815</td>\n",
       "      <td>(308)956-9340x849</td>\n",
       "      <td>246.0</td>\n",
       "      <td>4.224368e+15</td>\n",
       "      <td>Pharmacist, hospital</td>\n",
       "      <td>nicholaslong@example.org</td>\n",
       "      <td>http://www.washington.com/</td>\n",
       "      <td>Philippines</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Michael Andrews</td>\n",
       "      <td>...</td>\n",
       "      <td>2.322087</td>\n",
       "      <td>1.219705</td>\n",
       "      <td>0.713342</td>\n",
       "      <td>0.112852</td>\n",
       "      <td>4.809016</td>\n",
       "      <td>-1.250286</td>\n",
       "      <td>-0.021655</td>\n",
       "      <td>26.685279</td>\n",
       "      <td>1.674996</td>\n",
       "      <td>Worthlessness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139996</th>\n",
       "      <td>501612410</td>\n",
       "      <td>250.952.6242x324</td>\n",
       "      <td>5704.0</td>\n",
       "      <td>3.595100e+15</td>\n",
       "      <td>Chemical engineer</td>\n",
       "      <td>garciakimberly@example.com</td>\n",
       "      <td>https://barber.com/</td>\n",
       "      <td>Belarus</td>\n",
       "      <td>🧇</td>\n",
       "      <td>Luke Wagner</td>\n",
       "      <td>...</td>\n",
       "      <td>-11.413135</td>\n",
       "      <td>1.773445</td>\n",
       "      <td>-3.209504</td>\n",
       "      <td>7.831701</td>\n",
       "      <td>1.818230</td>\n",
       "      <td>6.058754</td>\n",
       "      <td>0.864957</td>\n",
       "      <td>30.435647</td>\n",
       "      <td>5.051781</td>\n",
       "      <td>Fatigue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139997</th>\n",
       "      <td>C50473932</td>\n",
       "      <td>217.970.6851</td>\n",
       "      <td>197.0</td>\n",
       "      <td>2.563124e+15</td>\n",
       "      <td>Educational psychologist</td>\n",
       "      <td>brandon71@example.org</td>\n",
       "      <td>http://haas-bryan.com/</td>\n",
       "      <td>Comoros</td>\n",
       "      <td>👨🏽‍🏫</td>\n",
       "      <td>Vincent Baker</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.233369</td>\n",
       "      <td>-1.917208</td>\n",
       "      <td>4.754519</td>\n",
       "      <td>-3.404189</td>\n",
       "      <td>1.013196</td>\n",
       "      <td>-2.941743</td>\n",
       "      <td>0.660514</td>\n",
       "      <td>16.354339</td>\n",
       "      <td>7.605151</td>\n",
       "      <td>Stress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139998</th>\n",
       "      <td>S27033224</td>\n",
       "      <td>747-866-0589</td>\n",
       "      <td>621.0</td>\n",
       "      <td>4.223086e+15</td>\n",
       "      <td>Town planner</td>\n",
       "      <td>salinassamantha@example.com</td>\n",
       "      <td>http://www.case-irwin.com/</td>\n",
       "      <td>Slovakia (Slovak Republic)</td>\n",
       "      <td>🤽🏼</td>\n",
       "      <td>James Smith</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.958600</td>\n",
       "      <td>-0.871512</td>\n",
       "      <td>3.122100</td>\n",
       "      <td>3.637444</td>\n",
       "      <td>3.373495</td>\n",
       "      <td>-4.646573</td>\n",
       "      <td>-3.040717</td>\n",
       "      <td>13.430366</td>\n",
       "      <td>-5.786846</td>\n",
       "      <td>Fatigue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139999</th>\n",
       "      <td>Z76517010</td>\n",
       "      <td>728-556-3128</td>\n",
       "      <td>538.0</td>\n",
       "      <td>3.021959e+13</td>\n",
       "      <td>Armed forces technical officer</td>\n",
       "      <td>jefferyday@example.org</td>\n",
       "      <td>http://estrada.com/</td>\n",
       "      <td>Albania</td>\n",
       "      <td>📶</td>\n",
       "      <td>Jesse Hernandez</td>\n",
       "      <td>...</td>\n",
       "      <td>-9.750174</td>\n",
       "      <td>2.568428</td>\n",
       "      <td>-31.601196</td>\n",
       "      <td>-2.340051</td>\n",
       "      <td>-2.862995</td>\n",
       "      <td>3.700141</td>\n",
       "      <td>-1.129484</td>\n",
       "      <td>-11.735526</td>\n",
       "      <td>8.360681</td>\n",
       "      <td>Worthlessness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140000 rows × 111 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              UID                ph_no     cvv  credit_card_number   \n",
       "0       T77962103           7059868985   673.0        2.131868e+14  \\\n",
       "1       O92591533  +1-288-810-2425x013   983.0        4.820055e+15   \n",
       "2       O31883571    791.502.4387x7276   855.0        4.031429e+15   \n",
       "3       M18080565           2522308761   806.0        3.036423e+13   \n",
       "4       Q74073854     683-521-2001x423   424.0        4.610207e+15   \n",
       "...           ...                  ...     ...                 ...   \n",
       "139995  363253815    (308)956-9340x849   246.0        4.224368e+15   \n",
       "139996  501612410     250.952.6242x324  5704.0        3.595100e+15   \n",
       "139997  C50473932         217.970.6851   197.0        2.563124e+15   \n",
       "139998  S27033224         747-866-0589   621.0        4.223086e+15   \n",
       "139999  Z76517010         728-556-3128   538.0        3.021959e+13   \n",
       "\n",
       "                                   job                        email   \n",
       "0                                 Copy    guerrerodavid@example.org  \\\n",
       "1        Interior and spatial designer       michaela47@example.org   \n",
       "2             Scientist, water quality         zhoffman@example.org   \n",
       "3                         Toxicologist           hprice@example.net   \n",
       "4                    Software engineer        michael61@example.net   \n",
       "...                                ...                          ...   \n",
       "139995            Pharmacist, hospital     nicholaslong@example.org   \n",
       "139996               Chemical engineer   garciakimberly@example.com   \n",
       "139997        Educational psychologist        brandon71@example.org   \n",
       "139998                    Town planner  salinassamantha@example.com   \n",
       "139999  Armed forces technical officer       jefferyday@example.org   \n",
       "\n",
       "                               url                     country  emoji   \n",
       "0               http://garcia.org/                       Gabon      📑  \\\n",
       "1                http://ortiz.com/                     Bermuda     🥷🏿   \n",
       "2                http://kelly.com/       Sao Tome and Principe    👩‍💻   \n",
       "3                http://mann.info/                       Korea    NaN   \n",
       "4        https://www.mitchell.biz/          Russian Federation  💆🏿‍♂️   \n",
       "...                            ...                         ...    ...   \n",
       "139995  http://www.washington.com/                 Philippines    NaN   \n",
       "139996         https://barber.com/                     Belarus      🧇   \n",
       "139997      http://haas-bryan.com/                     Comoros   👨🏽‍🏫   \n",
       "139998  http://www.case-irwin.com/  Slovakia (Slovak Republic)     🤽🏼   \n",
       "139999         http://estrada.com/                     Albania      📶   \n",
       "\n",
       "                   name  ...      qhUzJ      FpCOT      zEnW3     ASDn5   \n",
       "0          Cathy Cherry  ...   9.028569  -0.173742 -16.438254 -5.683709  \\\n",
       "1         Austin Graham  ... -40.773924   3.836528 -20.503862  4.345120   \n",
       "2           Frank Smith  ...  32.831181   6.673810  31.545004 -1.118935   \n",
       "3          Austin Young  ... -11.829455 -10.870125  26.131133  2.115912   \n",
       "4       Kimberly Wilson  ...  10.957430  -1.530905 -17.670061  0.132419   \n",
       "...                 ...  ...        ...        ...        ...       ...   \n",
       "139995  Michael Andrews  ...   2.322087   1.219705   0.713342  0.112852   \n",
       "139996      Luke Wagner  ... -11.413135   1.773445  -3.209504  7.831701   \n",
       "139997    Vincent Baker  ...  -1.233369  -1.917208   4.754519 -3.404189   \n",
       "139998      James Smith  ...  -0.958600  -0.871512   3.122100  3.637444   \n",
       "139999  Jesse Hernandez  ...  -9.750174   2.568428 -31.601196 -2.340051   \n",
       "\n",
       "           vF2is     pZijn     WUc3c      sCIyG      qaERi          state  \n",
       "0       4.495021  1.397634 -2.161601   7.619064  -7.002747        Despair  \n",
       "1      -0.199043  3.414072  8.673901  -0.635993 -11.039355  Worthlessness  \n",
       "2      -2.109316  6.773527  1.718408 -35.955251   8.675279         Stress  \n",
       "3       7.373067 -0.238810 -5.427878  24.501781  -0.845627        Fatigue  \n",
       "4       3.696047  4.588960 -3.735829  18.194147  10.404818          Tears  \n",
       "...          ...       ...       ...        ...        ...            ...  \n",
       "139995  4.809016 -1.250286 -0.021655  26.685279   1.674996  Worthlessness  \n",
       "139996  1.818230  6.058754  0.864957  30.435647   5.051781        Fatigue  \n",
       "139997  1.013196 -2.941743  0.660514  16.354339   7.605151         Stress  \n",
       "139998  3.373495 -4.646573 -3.040717  13.430366  -5.786846        Fatigue  \n",
       "139999 -2.862995  3.700141 -1.129484 -11.735526   8.360681  Worthlessness  \n",
       "\n",
       "[140000 rows x 111 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "def proc(data,test=False):\n",
    "    collist = data.columns[10:] if test else data.columns[10:-1]\n",
    "    for c in collist:\n",
    "        for r in range(data.shape[0]):\n",
    "            if not data.loc[r,c] in [np.nan,'nan',0.0]:\n",
    "                try:\n",
    "                    data.loc[r,c] = float(data.loc[r,c][6:-6])\n",
    "                except:\n",
    "                    print(\"exception at\",r,c)\n",
    "        print(f\"col : {col} done\")\n",
    "    return data\n",
    "\n",
    "if not processed:\n",
    "    data = proc(data)\n",
    "#     for c in collist[10:-1]:\n",
    "#         for r in range(rows):\n",
    "#             if not data.loc[r,c] in [np.nan, 'nan', 0.0]:\n",
    "#                 try:\n",
    "#                     data.loc[r,c] = float(data.loc[r,c][6:-6])\n",
    "#                 except:\n",
    "#                     print(\"exception at\",r,c)\n",
    "    #now to fill null values with mean\n",
    "#     data.fillna(data.mean())\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3e248a9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['K3Ll9', '19rjS', 'yeIIP', 'Bw1V5', '5k16L', 'e2l5S', 'cg31y', '8SVMv',\n",
      "       'Xsi3p', 'l8Y6n', 'vRJwh', 'CTzXJ', '13SMK', 'WUuos', 'UyaHP', 'i6ebQ',\n",
      "       'ciVeL', 'fOHiM', '1AFO4', 'b0QUS', 'vgNLa', 'EjWMB', 'Z9KvZ', 'LsnIb',\n",
      "       'kMpCl', 'gomep', '1Jfzv', 'piHSs', 'SQLeJ', '184Wa', 'HTOAr', 'gItG5',\n",
      "       'K7fG4', '9rzVt', '7Jilh', 'AOWfH', '7hdd4', '1QuOM', 'SQUqt', 'GZVIZ',\n",
      "       'TdpFZ', 'LeWc9', '6eCdH', 'tnrKz', 'YXZTK', 'pCgEW', 'hOqKS', 'Rc2TO',\n",
      "       'C3boy', 'NgYAu', 'pTU4e', 'ABLCt', 'bozM7', 'Got9C', 'OyOUV', 'lcIhR',\n",
      "       'aDdCa', 'UZRdX', 'aWUYS', 'eX7iy', 'tDFkL', 'eWqKh', 'zPoSs', 'XIIc6',\n",
      "       'QyiJU', 'aUxm5', 'ToGNB', 'BYbv6', 'VWwlJ', 'gl2KL', 'eWCdk', 'qLu1T',\n",
      "       'cO1Q3', 'X2hRu', 'sRaqu', 'dKQBH', 'NKspI', 'Ncx50', 'QfXOs', 'Zavi4',\n",
      "       'HpyWy', 'op6uG', 'k9NrC', 'dTj0P', '9IHv6', 'yiWZJ', 'JDyPw', '3nCgU',\n",
      "       'Bz7Ov', 'uN0aA', 'OaMqz', 'qhUzJ', 'FpCOT', 'zEnW3', 'ASDn5', 'vF2is',\n",
      "       'pZijn', 'WUc3c', 'sCIyG', 'qaERi'],\n",
      "      dtype='object')\n",
      "0               Despair\n",
      "1         Worthlessness\n",
      "2                Stress\n",
      "3               Fatigue\n",
      "4                 Tears\n",
      "              ...      \n",
      "139995    Worthlessness\n",
      "139996          Fatigue\n",
      "139997           Stress\n",
      "139998          Fatigue\n",
      "139999    Worthlessness\n",
      "Name: state, Length: 140000, dtype: object\n",
      "K3Ll9 :  0\n",
      "19rjS :  0\n",
      "yeIIP :  0\n",
      "Bw1V5 :  0\n",
      "5k16L :  0\n",
      "e2l5S :  0\n",
      "cg31y :  0\n",
      "8SVMv :  0\n",
      "Xsi3p :  0\n",
      "l8Y6n :  0\n",
      "vRJwh :  0\n",
      "CTzXJ :  0\n",
      "13SMK :  0\n",
      "WUuos :  0\n",
      "UyaHP :  0\n",
      "i6ebQ :  0\n",
      "ciVeL :  0\n",
      "fOHiM :  0\n",
      "1AFO4 :  0\n",
      "b0QUS :  0\n",
      "vgNLa :  0\n",
      "EjWMB :  0\n",
      "Z9KvZ :  0\n",
      "LsnIb :  0\n",
      "kMpCl :  0\n",
      "gomep :  0\n",
      "1Jfzv :  0\n",
      "piHSs :  0\n",
      "SQLeJ :  0\n",
      "184Wa :  0\n",
      "HTOAr :  0\n",
      "gItG5 :  0\n",
      "K7fG4 :  0\n",
      "9rzVt :  0\n",
      "7Jilh :  0\n",
      "AOWfH :  0\n",
      "7hdd4 :  0\n",
      "1QuOM :  0\n",
      "SQUqt :  0\n",
      "GZVIZ :  0\n",
      "TdpFZ :  0\n",
      "LeWc9 :  0\n",
      "6eCdH :  0\n",
      "tnrKz :  0\n",
      "YXZTK :  0\n",
      "pCgEW :  0\n",
      "hOqKS :  0\n",
      "Rc2TO :  0\n",
      "C3boy :  0\n",
      "NgYAu :  0\n",
      "pTU4e :  0\n",
      "ABLCt :  0\n",
      "bozM7 :  0\n",
      "Got9C :  0\n",
      "OyOUV :  0\n",
      "lcIhR :  0\n",
      "aDdCa :  0\n",
      "UZRdX :  0\n",
      "aWUYS :  0\n",
      "eX7iy :  0\n",
      "tDFkL :  0\n",
      "eWqKh :  0\n",
      "zPoSs :  0\n",
      "XIIc6 :  0\n",
      "QyiJU :  0\n",
      "aUxm5 :  0\n",
      "ToGNB :  0\n",
      "BYbv6 :  0\n",
      "VWwlJ :  0\n",
      "gl2KL :  0\n",
      "eWCdk :  0\n",
      "qLu1T :  0\n",
      "cO1Q3 :  0\n",
      "X2hRu :  0\n",
      "sRaqu :  0\n",
      "dKQBH :  0\n",
      "NKspI :  0\n",
      "Ncx50 :  0\n",
      "QfXOs :  0\n",
      "Zavi4 :  0\n",
      "HpyWy :  0\n",
      "op6uG :  0\n",
      "k9NrC :  0\n",
      "dTj0P :  0\n",
      "9IHv6 :  0\n",
      "yiWZJ :  0\n",
      "JDyPw :  0\n",
      "3nCgU :  0\n",
      "Bz7Ov :  0\n",
      "uN0aA :  0\n",
      "OaMqz :  0\n",
      "qhUzJ :  0\n",
      "FpCOT :  0\n",
      "zEnW3 :  0\n",
      "ASDn5 :  0\n",
      "vF2is :  0\n",
      "pZijn :  0\n",
      "WUc3c :  0\n",
      "sCIyG :  0\n",
      "qaERi :  0\n"
     ]
    }
   ],
   "source": [
    "data = data.drop(list(collist[:10]),axis=1)\n",
    "y = data[\"state\"]\n",
    "x = data.drop(\"state\",axis=1)\n",
    "\n",
    "x = x.fillna(x.mean())\n",
    "\n",
    "print(x.columns)\n",
    "#check:\n",
    "print(y)\n",
    "for col in x.columns:\n",
    "    print(col,\": \",x[col].isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f8f32946",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-20 16:01:11.128022: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-07-20 16:01:11.176039: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-07-20 16:01:11.176715: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-07-20 16:01:12.022988: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98000 98000 42000 42000\n",
      "(98000, 100)            K3Ll9      19rjS      yeIIP      Bw1V5     5k16L      e2l5S   \n",
      "59873  -0.877652 -27.692941  -7.579740  17.743760 -4.311822  -3.892605  \\\n",
      "111793 -1.945719   0.810767  22.253275 -18.658447 -6.217910  -4.878691   \n",
      "15352  -1.974151  -8.513553 -12.895982  34.387628  4.252077 -23.153682   \n",
      "72113  -2.128551  -4.330913 -12.924452 -26.858458  1.030194 -29.078196   \n",
      "100800 -2.658644  -2.497753 -19.229575 -14.609430 -4.416486   4.392323   \n",
      "...          ...        ...        ...        ...       ...        ...   \n",
      "16086   1.100286  -1.693430  19.462414  17.270923  5.310261  10.408747   \n",
      "91876  -8.697689   4.168451  29.123761   6.828033 -3.856062   0.709317   \n",
      "36628   5.230021  11.526027 -19.021854  -1.099045 -7.240480  -9.416289   \n",
      "55285  -1.192768 -19.598199  -9.770858  16.054999 -3.698670 -18.651240   \n",
      "599    -0.951892  -4.009965   6.344475  -8.534966 -0.404427  -9.071126   \n",
      "\n",
      "           cg31y      8SVMv     Xsi3p      l8Y6n  ...      OaMqz      qhUzJ   \n",
      "59873   3.484822  17.797053 -6.419908 -10.745853  ...  -3.562581 -19.822731  \\\n",
      "111793  3.589081  17.427490 -1.002925  21.500146  ...   2.415403   0.432674   \n",
      "15352   0.182164 -20.000417 -0.483371   9.033080  ...   4.696660  29.776818   \n",
      "72113   6.729233 -18.914469  3.716503 -20.029315  ...  13.564326  44.149578   \n",
      "100800  0.168142 -16.933068  1.142102  -2.073146  ...  -3.066117  19.647210   \n",
      "...          ...        ...       ...        ...  ...        ...        ...   \n",
      "16086  -4.770287  29.826382 -3.223935   2.220533  ...  24.967743 -31.010725   \n",
      "91876  -0.773507  11.397889 -1.435660  13.211723  ... -20.021939 -17.291217   \n",
      "36628  -3.776170  32.964299  7.391624 -12.125768  ... -29.077281  -8.581868   \n",
      "55285  -0.545550   3.413897 -2.346488   6.284256  ... -13.110555  -8.704818   \n",
      "599     6.271864   7.747241 -9.972965  -1.413273  ... -11.910230  -3.031375   \n",
      "\n",
      "           FpCOT      zEnW3     ASDn5     vF2is     pZijn     WUc3c   \n",
      "59873  -3.071588   5.287227 -0.001276 -0.048720 -1.261040 -0.669073  \\\n",
      "111793 -0.755334  -0.744673 -0.117939  0.894912 -0.345430  0.355622   \n",
      "15352   1.102785  -1.988105  2.103359  2.862308  9.467222 -0.388640   \n",
      "72113   2.586930  15.171539 -3.866367 -1.030959  0.783692  1.390466   \n",
      "100800  2.985012  -6.370845  2.641023 -0.202851 -0.077920 -2.847050   \n",
      "...          ...        ...       ...       ...       ...       ...   \n",
      "16086   3.843565  18.065430 -1.507305  4.462548  6.500696  3.212814   \n",
      "91876  -0.666369  -9.735395 -7.924379  4.365795  0.416147  4.324701   \n",
      "36628  -3.492791 -16.290731 -0.593228 -0.793344 -5.192636  1.604088   \n",
      "55285  -7.113628   8.483615 -0.173205 -0.469898 -2.675032  6.938033   \n",
      "599    -4.380961  -9.265692 -7.972333 -0.272986 -0.653363 -1.981405   \n",
      "\n",
      "            sCIyG     qaERi  \n",
      "59873    2.062069 -2.748227  \n",
      "111793 -18.770911 -6.127253  \n",
      "15352    7.354212  5.458680  \n",
      "72113  -12.842309  2.276057  \n",
      "100800 -10.395609 -3.761108  \n",
      "...           ...       ...  \n",
      "16086    6.151399 -2.199423  \n",
      "91876    0.982778  5.253953  \n",
      "36628  -11.215467 -3.011870  \n",
      "55285   -3.945655 -8.926996  \n",
      "599    -10.373855 -0.006920  \n",
      "\n",
      "[98000 rows x 100 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "59873    -0.877652\n",
       "111793   -1.945719\n",
       "15352    -1.974151\n",
       "72113    -2.128551\n",
       "100800   -2.658644\n",
       "            ...   \n",
       "16086     1.100286\n",
       "91876    -8.697689\n",
       "36628     5.230021\n",
       "55285    -1.192768\n",
       "599      -0.951892\n",
       "Name: K3Ll9, Length: 98000, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder as OH\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y = y.values.reshape(-1,1)\n",
    "enc = OH(sparse_output=False)\n",
    "y = enc.fit_transform(y)\n",
    "# le = LE()\n",
    "# y = le.fit_transform(y)\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3)\n",
    "print(len(x_train),len(y_train),len(x_test),len(y_test))\n",
    "print(x_train.shape,x_train)\n",
    "x_train[collist[10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a91ff08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_train = (x_train-x_train.min())/(x_train.max()-x_train.min())\n",
    "# x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d14222e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "inp_shape = x_train.shape[1]\n",
    "out_shape = y_train.shape[1]\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.Input(shape=(inp_shape,)),\n",
    "    tf.keras.layers.Dense(80,activation='relu'),\n",
    "    tf.keras.layers.Dense(75,activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.2),\n",
    "    tf.keras.layers.Dense(15,activation='relu'),\n",
    "#     tf.keras.layers.Dense(32,activation='relu'),\n",
    "    tf.keras.layers.Dense(out_shape, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "df3f39d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "602f6b6a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8eca1b80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 1.9934 - accuracy: 0.2677 - val_loss: 1.4115 - val_accuracy: 0.5122\n",
      "Epoch 2/10\n",
      "11025/11025 [==============================] - 15s 1ms/step - loss: 1.1823 - accuracy: 0.6093 - val_loss: 0.8885 - val_accuracy: 0.7179\n",
      "Epoch 3/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.8617 - accuracy: 0.7290 - val_loss: 0.7049 - val_accuracy: 0.7812\n",
      "Epoch 4/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.7295 - accuracy: 0.7775 - val_loss: 0.5897 - val_accuracy: 0.8274\n",
      "Epoch 5/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.6547 - accuracy: 0.8038 - val_loss: 0.5609 - val_accuracy: 0.8324\n",
      "Epoch 6/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.6018 - accuracy: 0.8218 - val_loss: 0.5180 - val_accuracy: 0.8488\n",
      "Epoch 7/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.5694 - accuracy: 0.8332 - val_loss: 0.5087 - val_accuracy: 0.8540\n",
      "Epoch 8/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.5344 - accuracy: 0.8440 - val_loss: 0.4575 - val_accuracy: 0.8676\n",
      "Epoch 9/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.5149 - accuracy: 0.8522 - val_loss: 0.4788 - val_accuracy: 0.8610\n",
      "Epoch 10/10\n",
      "11025/11025 [==============================] - 16s 1ms/step - loss: 0.4993 - accuracy: 0.8574 - val_loss: 0.4356 - val_accuracy: 0.8772\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f435a709990>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 8\n",
    "epochs = 10\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac88a1b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1313/1313 [==============================] - 2s 1ms/step - loss: 0.4398 - accuracy: 0.8770\n",
      "Test loss: 0.43984296917915344, Test accuracy: 0.877047598361969\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy = model.evaluate(x_test, y_test)\n",
    "print(f'Test loss: {loss}, Test accuracy: {accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "54220844",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = tf.convert_to_tensor(data.loc[0,collist[10:-1]].values.reshape(1,-1))\n",
    "# # model.call(x)\n",
    "# # print(x_train[collist[10]].info)\n",
    "# x = data.loc[0,collist[10:-1]]\n",
    "# # x = [9.0]*inp_shape\n",
    "# x = np.array(x)\n",
    "# x = x.reshape(1,-1)\n",
    "# x = x.astype(np.float64)  # Or np.float64 if that was the data type used during training\n",
    "\n",
    "# model.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d22730ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             UID                   ph_no     cvv  credit_card_number   \n",
      "0      675919160   001-869-364-3240x1461   632.0        4.787566e+12  \\\n",
      "1      V09461652     +1-573-696-9623x435   548.0        4.804074e+15   \n",
      "2      S75396644    001-755-901-1494x000   808.0        4.890170e+15   \n",
      "3      598599835       (625)805-7487x931  9468.0        5.428366e+15   \n",
      "4      W60397022      556-206-9662x97397   647.0        2.131239e+14   \n",
      "...          ...                     ...     ...                 ...   \n",
      "59995  G18996472  001-354-705-4724x96174   546.0        2.131534e+14   \n",
      "59996  119545284       541.850.1553x8339   248.0        3.479339e+14   \n",
      "59997  Q57447995      936.930.2490x92938   446.0        1.800868e+14   \n",
      "59998  611053577            326.820.7743   446.0        3.674868e+13   \n",
      "59999  C13097269            393.924.0554   653.0        2.131188e+14   \n",
      "\n",
      "                                         job                         email   \n",
      "0                Scientist, research (maths)  lawrencereginald@example.org  \\\n",
      "1           Higher education careers adviser        jennifer41@example.org   \n",
      "2                                Illustrator      valdezsheryl@example.net   \n",
      "3                     Audiological scientist                           NaN   \n",
      "4      Development worker, international aid    jefferyrussell@example.org   \n",
      "...                                      ...                           ...   \n",
      "59995                           Psychiatrist         michael80@example.org   \n",
      "59996                            Chiropodist      sandragarner@example.com   \n",
      "59997                             Bookseller           yharris@example.com   \n",
      "59998                 Health service manager      tammywallace@example.org   \n",
      "59999                  Editor, commissioning      jefferybarry@example.net   \n",
      "\n",
      "                                   url                           country   \n",
      "0                   http://walker.org/                          Honduras  \\\n",
      "1                     http://shah.com/              Netherlands Antilles   \n",
      "2                  http://barrett.com/  Lao People's Democratic Republic   \n",
      "3                 http://melendez.com/                      Sierra Leone   \n",
      "4                    http://hicks.net/                     Faroe Islands   \n",
      "...                                ...                               ...   \n",
      "59995  http://www.martinez-rogers.com/                              Cuba   \n",
      "59996           http://www.brooks.com/                          Kiribati   \n",
      "59997     https://www.white-green.com/                             Benin   \n",
      "59998           https://www.rojas.com/                           Liberia   \n",
      "59999              http://miller.info/      United States Virgin Islands   \n",
      "\n",
      "       emoji               name  ...      OaMqz      qhUzJ     FpCOT   \n",
      "0      🧜🏻‍♀️       Jean Barajas  ... -21.248937  -5.726684  1.577796  \\\n",
      "1      💁🏻‍♀️       Sandra Ellis  ... -33.934399 -19.149704 -2.868718   \n",
      "2         👭🏼    Elizabeth Smith  ...  12.391218 -14.048287 -5.027959   \n",
      "3         🖲️  Allison Rodriguez  ...  11.456072 -12.459837 -3.448771   \n",
      "4         🏌️       Helen Howell  ...  19.865807 -24.537809  1.106189   \n",
      "...      ...                ...  ...        ...        ...       ...   \n",
      "59995   👩🏿‍🦯     Shawn Reynolds  ...  19.649591  -8.807412  8.620351   \n",
      "59996  🤷🏼‍♂️       Jason Turner  ...        NaN -39.585549  2.507032   \n",
      "59997   👩🏾‍🏭      Melissa Cooke  ...  32.368422 -19.829218  5.767188   \n",
      "59998      🫁       Mariah Stone  ... -17.347608  14.993694  2.471468   \n",
      "59999   🧗‍♀️      Nicole Murphy  ...   9.144159 -15.927747  1.628786   \n",
      "\n",
      "           zEnW3     ASDn5     vF2is     pZijn     WUc3c      sCIyG     qaERi  \n",
      "0      -8.911004  5.981008 -1.297894 -1.673945  1.446459 -29.421645  1.795004  \n",
      "1      15.453724 -6.211322 -0.933979  0.001376 -4.159291  -1.540406  1.296943  \n",
      "2       6.754527  7.670390  0.347081  1.407364  6.091721 -22.070410 -3.748772  \n",
      "3      20.407682 -1.601812  1.355095  0.979308 -1.409996   2.120580 -3.924106  \n",
      "4     -14.192347  3.930237  9.832580 -0.672717 -7.779142  15.756719 -7.693867  \n",
      "...          ...       ...       ...       ...       ...        ...       ...  \n",
      "59995  -1.111601  2.367149 -1.469333  1.480501  2.712732  -8.616272 -7.006872  \n",
      "59996   1.835891 -0.362596  3.059471  3.154485  1.802282   4.723408 -4.572501  \n",
      "59997 -18.795186 -9.603047 -3.901352  3.412305  4.303302 -23.636226  0.553597  \n",
      "59998 -23.408356 -3.488413 -5.711458  2.541118 -3.005590   4.464814 -2.205268  \n",
      "59999 -16.504059  2.623832 -5.564953 -2.776443  8.154846 -41.071138 -0.330314  \n",
      "\n",
      "[60000 rows x 110 columns]\n",
      "(60000, 110)\n",
      "K3Ll9 :  0\n",
      "19rjS :  0\n",
      "yeIIP :  0\n",
      "Bw1V5 :  0\n",
      "5k16L :  0\n",
      "e2l5S :  0\n",
      "cg31y :  0\n",
      "8SVMv :  0\n",
      "Xsi3p :  0\n",
      "l8Y6n :  0\n",
      "vRJwh :  0\n",
      "CTzXJ :  0\n",
      "13SMK :  0\n",
      "WUuos :  0\n",
      "UyaHP :  0\n",
      "i6ebQ :  0\n",
      "ciVeL :  0\n",
      "fOHiM :  0\n",
      "1AFO4 :  0\n",
      "b0QUS :  0\n",
      "vgNLa :  0\n",
      "EjWMB :  0\n",
      "Z9KvZ :  0\n",
      "LsnIb :  0\n",
      "kMpCl :  0\n",
      "gomep :  0\n",
      "1Jfzv :  0\n",
      "piHSs :  0\n",
      "SQLeJ :  0\n",
      "184Wa :  0\n",
      "HTOAr :  0\n",
      "gItG5 :  0\n",
      "K7fG4 :  0\n",
      "9rzVt :  0\n",
      "7Jilh :  0\n",
      "AOWfH :  0\n",
      "7hdd4 :  0\n",
      "1QuOM :  0\n",
      "SQUqt :  0\n",
      "GZVIZ :  0\n",
      "TdpFZ :  0\n",
      "LeWc9 :  0\n",
      "6eCdH :  0\n",
      "tnrKz :  0\n",
      "YXZTK :  0\n",
      "pCgEW :  0\n",
      "hOqKS :  0\n",
      "Rc2TO :  0\n",
      "C3boy :  0\n",
      "NgYAu :  0\n",
      "pTU4e :  0\n",
      "ABLCt :  0\n",
      "bozM7 :  0\n",
      "Got9C :  0\n",
      "OyOUV :  0\n",
      "lcIhR :  0\n",
      "aDdCa :  0\n",
      "UZRdX :  0\n",
      "aWUYS :  0\n",
      "eX7iy :  0\n",
      "tDFkL :  0\n",
      "eWqKh :  0\n",
      "zPoSs :  0\n",
      "XIIc6 :  0\n",
      "QyiJU :  0\n",
      "aUxm5 :  0\n",
      "ToGNB :  0\n",
      "BYbv6 :  0\n",
      "VWwlJ :  0\n",
      "gl2KL :  0\n",
      "eWCdk :  0\n",
      "qLu1T :  0\n",
      "cO1Q3 :  0\n",
      "X2hRu :  0\n",
      "sRaqu :  0\n",
      "dKQBH :  0\n",
      "NKspI :  0\n",
      "Ncx50 :  0\n",
      "QfXOs :  0\n",
      "Zavi4 :  0\n",
      "HpyWy :  0\n",
      "op6uG :  0\n",
      "k9NrC :  0\n",
      "dTj0P :  0\n",
      "9IHv6 :  0\n",
      "yiWZJ :  0\n",
      "JDyPw :  0\n",
      "3nCgU :  0\n",
      "Bz7Ov :  0\n",
      "uN0aA :  0\n",
      "OaMqz :  0\n",
      "qhUzJ :  0\n",
      "FpCOT :  0\n",
      "zEnW3 :  0\n",
      "ASDn5 :  0\n",
      "vF2is :  0\n",
      "pZijn :  0\n",
      "WUc3c :  0\n",
      "sCIyG :  0\n",
      "qaERi :  0\n"
     ]
    }
   ],
   "source": [
    "predset = pd.read_csv(\"data/test_processed.csv\")\n",
    "print(predset)\n",
    "\n",
    "print(predset.shape)\n",
    "x_pred = predset.drop(predset.columns[:10],axis=1)\n",
    "x_pred.fillna(x_pred.mean(),inplace=True)\n",
    "for col in x_pred:\n",
    "    print(col,\": \",x_pred[col].isna().sum())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "270bda13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1875/1875 [==============================] - 2s 964us/step\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "73f2a03d",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = [e[3:] for e in enc.get_feature_names_out()]\n",
    "preds = [np.argmax(x) for x in predictions]\n",
    "pred_t = [classes[i] for i in preds]\n",
    "out = pd.DataFrame()\n",
    "out[\"UID\"] = predset[\"UID\"]\n",
    "out[\"state\"] = pred_t\n",
    "out.to_csv(\"data/preds.csv\",index=False)\n",
    "# out[\"state\"] = [classes[i] for i in preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "607190bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 2)\n",
      "             UID    state\n",
      "0      675919160   Nausea\n",
      "1      V09461652   Stress\n",
      "2      S75396644  Anxiety\n",
      "3      598599835    Tears\n",
      "4      W60397022    Dread\n",
      "...          ...      ...\n",
      "59995  G18996472    Dread\n",
      "59996  119545284  Fatigue\n",
      "59997  Q57447995    Tears\n",
      "59998  611053577   Stress\n",
      "59999  C13097269    Tears\n",
      "\n",
      "[60000 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print(out.shape)\n",
    "print(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}