UnbalanceData.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361634\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "from sklearn.utils import shuffle\n",
    "import pandas as pd\n",
    "\n",
    "with open( '/share/hel/datasets/fs/nlp/text_classification_bias/bios/BIOS.pkl', 'rb') as file:\n",
    "    bios_dicts = pickle.load(file)\n",
    "df = pd.DataFrame(bios_dicts)\n",
    "\n",
    "df.drop(['path','name','start_pos','URI','title'], axis=1, inplace=True)\n",
    "df['raw_title'] = df['raw_title'].map(lambda x: x.lower())\n",
    "raw_bio =[]\n",
    "bio = []\n",
    "for i in range(len(df)):\n",
    "    raw_bio.append(df['raw'][i].replace(df['raw_title'][i],'_'))\n",
    "    bio.append(df['bio'][i].replace(df['raw_title'][i],'_'))\n",
    "df['raw'] = raw_bio\n",
    "df['bio'] = bio\n",
    "print(len(df))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "903011\n",
      "2109\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "path = '/share/hel/datasets/jobiqo/talent.com/UK/'\n",
    "all_files = glob.glob(path + \"/*.csv\")\n",
    "\n",
    "li = []\n",
    "\n",
    "for filename in all_files:\n",
    "    frame = pd.read_csv(filename, index_col=None, header=0)\n",
    "    li.append(frame)\n",
    "\n",
    "uk_data = pd.concat(li, axis=0, ignore_index=True)\n",
    "uk_data.drop(['company','city','state','country','date','time','jobid'], axis=1, inplace=True)\n",
    "uk_data['title'] = uk_data['title'].map(lambda x: x.lower())\n",
    "print(len(uk_data))\n",
    "uk_data.drop_duplicates(subset=\"description\",  keep=False, inplace=True)\n",
    "new_uk_data = uk_data[uk_data['title'].isin(df.raw_title.unique())]\n",
    "print(len(new_uk_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/deepak/ipykernel_64679/462123052.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_uk_data['description'] = new_uk_data['description'].replace('\\n',' ', regex=True)\n"
     ]
    }
   ],
   "source": [
    "new_uk_data['description'] = new_uk_data['description'].replace('\\n',' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "software engineer           522\n",
       "senior software engineer    405\n",
       "dentist                     279\n",
       "accountant                  162\n",
       "teacher                     152\n",
       "architect                   146\n",
       "nurse                       141\n",
       "paralegal                    96\n",
       "painter                      70\n",
       "psychologist                 33\n",
       "personal trainer             23\n",
       "dietitian                    22\n",
       "interior designer            18\n",
       "photographer                 16\n",
       "psychotherapist               4\n",
       "journalist                    4\n",
       "assistant professor           4\n",
       "attorney                      3\n",
       "associate professor           3\n",
       "yoga instructor               2\n",
       "model                         1\n",
       "surgeon                       1\n",
       "physician                     1\n",
       "nutritionist                  1\n",
       "Name: title, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_uk_data['title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_remove = ['physician', 'nutritionist', 'surgeon', 'model', 'yoga instructor', 'attorney', 'associate professor', 'journalist', 'assistant professor', 'psychotherapist' ]\n",
    "\n",
    "for title in titles_remove:\n",
    "    new_uk_data = new_uk_data.drop(new_uk_data[new_uk_data['title']==title].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "software engineer           522\n",
       "senior software engineer    405\n",
       "dentist                     279\n",
       "accountant                  162\n",
       "teacher                     152\n",
       "architect                   146\n",
       "nurse                       141\n",
       "paralegal                    96\n",
       "painter                      70\n",
       "psychologist                 33\n",
       "personal trainer             23\n",
       "dietitian                    22\n",
       "interior designer            18\n",
       "photographer                 16\n",
       "Name: title, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_uk_data['title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>raw_title</th>\n",
       "      <th>gender</th>\n",
       "      <th>bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>* Nora Fisher Onar is an _ of international re...</td>\n",
       "      <td>assistant professor</td>\n",
       "      <td>F</td>\n",
       "      <td>_ is also a Ronald D. Asmus Policy Entrepreneu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dr. Shawn Zelig Aster is an _ of Bible at Yesh...</td>\n",
       "      <td>assistant professor</td>\n",
       "      <td>M</td>\n",
       "      <td>Dr. _ has held teaching and research positions...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Brittany McCandless is a _ with a print and br...</td>\n",
       "      <td>journalist</td>\n",
       "      <td>F</td>\n",
       "      <td>_ currently works on CNN’s newest primetime sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Arturo Toledo is an Architect. He runs a bouti...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>_ runs a boutique design studio attending clie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jan De Clercq is a solution _ working for the ...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>_ focuses on cloud security, identity and acce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361629</th>\n",
       "      <td>Fernando Quesada is an Attorney at-law admitte...</td>\n",
       "      <td>attorney</td>\n",
       "      <td>M</td>\n",
       "      <td>_ focuses _ practice in all forms of Energy (i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361630</th>\n",
       "      <td>Jeffrey Kucik is an _ in the School of Governm...</td>\n",
       "      <td>assistant professor</td>\n",
       "      <td>M</td>\n",
       "      <td>_ maintains the trade policy site www.trademon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361631</th>\n",
       "      <td>Leslie E. Seawright is an Assistant Professor ...</td>\n",
       "      <td>assistant professor</td>\n",
       "      <td>F</td>\n",
       "      <td>_ teaches courses in Technical Writing, Compos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361632</th>\n",
       "      <td>Paul Bird is a Chartered Accountant with over ...</td>\n",
       "      <td>accountant</td>\n",
       "      <td>M</td>\n",
       "      <td>_ has been responsible for many facets of fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361633</th>\n",
       "      <td>Jesse Cunha is an Associate Professor in the G...</td>\n",
       "      <td>associate professor</td>\n",
       "      <td>M</td>\n",
       "      <td>_ completed _ Ph.D. in Economics at Stanford U...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>320038 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      raw  \\\n",
       "0       * Nora Fisher Onar is an _ of international re...   \n",
       "2       Dr. Shawn Zelig Aster is an _ of Bible at Yesh...   \n",
       "3       Brittany McCandless is a _ with a print and br...   \n",
       "4       Arturo Toledo is an Architect. He runs a bouti...   \n",
       "5       Jan De Clercq is a solution _ working for the ...   \n",
       "...                                                   ...   \n",
       "361629  Fernando Quesada is an Attorney at-law admitte...   \n",
       "361630  Jeffrey Kucik is an _ in the School of Governm...   \n",
       "361631  Leslie E. Seawright is an Assistant Professor ...   \n",
       "361632  Paul Bird is a Chartered Accountant with over ...   \n",
       "361633  Jesse Cunha is an Associate Professor in the G...   \n",
       "\n",
       "                  raw_title gender  \\\n",
       "0       assistant professor      F   \n",
       "2       assistant professor      M   \n",
       "3                journalist      F   \n",
       "4                 architect      M   \n",
       "5                 architect      M   \n",
       "...                     ...    ...   \n",
       "361629             attorney      M   \n",
       "361630  assistant professor      M   \n",
       "361631  assistant professor      F   \n",
       "361632           accountant      M   \n",
       "361633  associate professor      M   \n",
       "\n",
       "                                                      bio  \n",
       "0       _ is also a Ronald D. Asmus Policy Entrepreneu...  \n",
       "2       Dr. _ has held teaching and research positions...  \n",
       "3       _ currently works on CNN’s newest primetime sh...  \n",
       "4       _ runs a boutique design studio attending clie...  \n",
       "5       _ focuses on cloud security, identity and acce...  \n",
       "...                                                   ...  \n",
       "361629  _ focuses _ practice in all forms of Energy (i...  \n",
       "361630  _ maintains the trade policy site www.trademon...  \n",
       "361631  _ teaches courses in Technical Writing, Compos...  \n",
       "361632  _ has been responsible for many facets of fina...  \n",
       "361633  _ completed _ Ph.D. in Economics at Stanford U...  \n",
       "\n",
       "[320038 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = df[df['raw_title'].isin(uk_data.title.unique())]\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>raw_title</th>\n",
       "      <th>gender</th>\n",
       "      <th>bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Arturo Toledo is an Architect. He runs a bouti...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>_ runs a boutique design studio attending clie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jan De Clercq is a solution _ working for the ...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>_ focuses on cloud security, identity and acce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Shane Lavalette is a _ currently living in Som...</td>\n",
       "      <td>photographer</td>\n",
       "      <td>M</td>\n",
       "      <td>_ received _ BFA from Tufts University in part...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Edmund J. Bourne, PhD, is a _ in northern Cali...</td>\n",
       "      <td>psychologist</td>\n",
       "      <td>M</td>\n",
       "      <td>_ is author of several books, including the be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Lama Kali Ma is a _ in the rare lineage of Bud...</td>\n",
       "      <td>teacher</td>\n",
       "      <td>F</td>\n",
       "      <td>_ teachings get straight to the heart of Tantr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361611</th>\n",
       "      <td>Benjamin Filarski is a French/Polish _ (b.1993...</td>\n",
       "      <td>photographer</td>\n",
       "      <td>M</td>\n",
       "      <td>_ studied sociology and political sciences at ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361614</th>\n",
       "      <td>Alice Sumo is a respected _ in Liberia, where ...</td>\n",
       "      <td>nurse</td>\n",
       "      <td>F</td>\n",
       "      <td>In _ three-decade career, _ has seen two civil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361616</th>\n",
       "      <td>Matthias Naab is a software _ at the Fraunhofe...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>_ is responsible for the development of _ure m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361622</th>\n",
       "      <td>Marina V. Umali is an Interior Designer &amp; Feng...</td>\n",
       "      <td>interior designer</td>\n",
       "      <td>F</td>\n",
       "      <td>_ creates custom solutions for _ clients using...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361632</th>\n",
       "      <td>Paul Bird is a Chartered Accountant with over ...</td>\n",
       "      <td>accountant</td>\n",
       "      <td>M</td>\n",
       "      <td>_ has been responsible for many facets of fina...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>112385 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      raw          raw_title  \\\n",
       "4       Arturo Toledo is an Architect. He runs a bouti...          architect   \n",
       "5       Jan De Clercq is a solution _ working for the ...          architect   \n",
       "6       Shane Lavalette is a _ currently living in Som...       photographer   \n",
       "7       Edmund J. Bourne, PhD, is a _ in northern Cali...       psychologist   \n",
       "10      Lama Kali Ma is a _ in the rare lineage of Bud...            teacher   \n",
       "...                                                   ...                ...   \n",
       "361611  Benjamin Filarski is a French/Polish _ (b.1993...       photographer   \n",
       "361614  Alice Sumo is a respected _ in Liberia, where ...              nurse   \n",
       "361616  Matthias Naab is a software _ at the Fraunhofe...          architect   \n",
       "361622  Marina V. Umali is an Interior Designer & Feng...  interior designer   \n",
       "361632  Paul Bird is a Chartered Accountant with over ...         accountant   \n",
       "\n",
       "       gender                                                bio  \n",
       "4           M  _ runs a boutique design studio attending clie...  \n",
       "5           M  _ focuses on cloud security, identity and acce...  \n",
       "6           M  _ received _ BFA from Tufts University in part...  \n",
       "7           M  _ is author of several books, including the be...  \n",
       "10          F  _ teachings get straight to the heart of Tantr...  \n",
       "...       ...                                                ...  \n",
       "361611      M  _ studied sociology and political sciences at ...  \n",
       "361614      F  In _ three-decade career, _ has seen two civil...  \n",
       "361616      M  _ is responsible for the development of _ure m...  \n",
       "361622      F  _ creates custom solutions for _ clients using...  \n",
       "361632      M  _ has been responsible for many facets of fina...  \n",
       "\n",
       "[112385 rows x 4 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles_remove = ['physician', 'nutritionist', 'surgeon', 'model', 'yoga instructor', 'attorney', 'associate professor', 'journalist', 'assistant professor', 'psychotherapist' ]\n",
    "\n",
    "for title in titles_remove:\n",
    "    new_df = new_df.drop(new_df[new_df['raw_title']==title].index)\n",
    "\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/bios.pkl', 'wb') as f:\n",
    "    pickle.dump(new_df, f)\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'wb') as f:\n",
    "    pickle.dump(new_uk_data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BIOS counterfactual and cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#profession gender ratio \n",
    "\n",
    "gender_ratio = {\"architect\" :                 (69,31),\n",
    "\"photographer\":                (85,15),\n",
    "\"psychologist\":                (20,80),\n",
    "\"teacher\":                     (25,75),\n",
    "\"nurse\":                       (11,89),\n",
    "\"software engineer\":           (84,16),\n",
    "\"painter\":                     (68,32),\n",
    "\"personal trainer\":            (65,35),\n",
    "\"dietitian\":                   (6,94),\n",
    "\"dentist\":                     (46,54),\n",
    "\"interior designer\":           (17,83),\n",
    "\"senior software engineer\":    (90,10),\n",
    "\"accountant\":                  (55,45),\n",
    "\"paralegal\":\t\t\t     (37,63) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 112385/112385 [00:49<00:00, 2292.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architect\n",
      "photographer\n",
      "psychologist\n",
      "teacher\n",
      "nurse\n",
      "software engineer\n",
      "painter\n",
      "personal trainer\n",
      "dietitian\n",
      "dentist\n",
      "interior designer\n",
      "senior software engineer\n",
      "accountant\n",
      "paralegal\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/bios.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "bios = pd.DataFrame(dicts)\n",
    "#bios = pd.DataFrame(dicts).reset_index()\n",
    "#bios = bios.drop('index', axis=1)\n",
    "\n",
    "with open( '/share/hel/datasets/fs/nlp/text_classification_bias/bios/BIOS.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "original_bios = pd.DataFrame(dicts)\n",
    "\n",
    "import regex as re\n",
    "from tqdm.autonotebook import tqdm\n",
    "\n",
    "male_words = ['mr','his','he','him','himself']\n",
    "female_words = ['mrs','hers','she','her','herself']\n",
    "\n",
    "bio_list = []\n",
    "counterfactual_list = []\n",
    "\n",
    "for id in tqdm(bios.index):\n",
    "    bio_string = bios['raw'][id]\n",
    "    name_list = list(original_bios['name'][id])\n",
    "    if len(name_list[1])==1:\n",
    "        name_list[1] = name_list[1]+'.'\n",
    "    if '' in name_list:\n",
    "        name_list.remove('')\n",
    "\n",
    "\n",
    "    bio_string = re.sub(' '.join(name_list), 'NAME',bio_string,flags=re.I)\n",
    "    for name in name_list:\n",
    "            bio_string = re.sub(r'\\s'+name+r'\\s', ' NAME ',bio_string,flags=re.I)\n",
    "    if bios['gender'][id]=='M':\n",
    "        bio_string = re.sub('NAME', 'Bob',bio_string)\n",
    "        bio_list.append(bio_string)\n",
    "        for i in range(len(male_words)):\n",
    "            bio_string = re.sub(r'\\s'+male_words[i]+r'\\s', ' '+female_words[i]+' ',bio_string,flags=re.I)\n",
    "\n",
    "        bio_string = re.sub('Bob', 'Alice',bio_string)\n",
    "    elif bios['gender'][id]=='F':\n",
    "        bio_string = re.sub('NAME', 'Alice',bio_string)\n",
    "        bio_list.append(bio_string)\n",
    "        for i in range(len(male_words)):\n",
    "            bio_string = re.sub(r'\\s'+female_words[i]+r'\\s',' '+ male_words[i]+' ',bio_string,flags=re.I)\n",
    "\n",
    "        bio_string = re.sub('Alice', 'Bob',bio_string)\n",
    "    counterfactual_list.append(bio_string)\n",
    "\n",
    "bios['bio'] = bio_list\n",
    "bios['counter_bio'] = counterfactual_list\n",
    "\n",
    "\n",
    "bios = bios.reset_index()\n",
    "bios = bios.drop('index', axis=1)\n",
    "\n",
    "unbalanced=\"True\"\n",
    "i=0\n",
    "for job in bios['raw_title'].unique():\n",
    "    print(job)\n",
    "    if unbalanced==\"True\":\n",
    "        if i==0:\n",
    "            new_bios = pd.concat([bios[(bios['gender']=='F') & (bios['raw_title']==job)].sample(n=2*gender_ratio[job][1],random_state=1),bios[(bios['gender']=='M') & (bios['raw_title']==job)].sample(n=2*gender_ratio[job][0],random_state=1)])\n",
    "        else:\n",
    "            new_bios = pd.concat([new_bios, pd.concat([bios[(bios['gender']=='F') & (bios['raw_title']==job)].sample(n=2*gender_ratio[job][1],random_state=1),bios[(bios['gender']=='M') & (bios['raw_title']==job)].sample(n=2*gender_ratio[job][0],random_state=1)])])    \n",
    "    else:\n",
    "        if i==0:\n",
    "            new_bios = pd.concat([bios[(bios['gender']=='F') & (bios['raw_title']==job)].sample(n=100,random_state=1),bios[(bios['gender']=='M') & (bios['raw_title']==job)].sample(n=100,random_state=1)])\n",
    "        else:\n",
    "            new_bios = pd.concat([new_bios, pd.concat([bios[(bios['gender']=='F') & (bios['raw_title']==job)].sample(n=100,random_state=1),bios[(bios['gender']=='M') & (bios['raw_title']==job)].sample(n=100,random_state=1)])])\n",
    "    i=i+1\n",
    "\n",
    "new_bios = new_bios.reset_index()\n",
    "new_bios = new_bios.drop('index', axis=1)\n",
    "\n",
    "anonymous_bio = []\n",
    "anonymous_counter_bio = []\n",
    "for i in range(len(new_bios)):\n",
    "    anonymous_bio.append(re.sub(new_bios['raw_title'][i], '_', new_bios['bio'][i],flags=re.I))\n",
    "    anonymous_counter_bio.append(re.sub(new_bios['raw_title'][i], '_', new_bios['counter_bio'][i],flags=re.I))\n",
    "\n",
    "new_bios['bio'] = anonymous_bio\n",
    "new_bios['counter_bio'] = anonymous_counter_bio\n",
    "\n",
    "new_bios = new_bios.reset_index()\n",
    "new_bios = new_bios.drop('index', axis=1)\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/new_bios_unbalanced.pkl', 'wb') as f:\n",
    "    pickle.dump(new_bios, f)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/new_bios_unbalanced.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "bios = pd.DataFrame(dicts).reset_index()\n",
    "bios.drop('index', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "architect                   200\n",
       "photographer                200\n",
       "psychologist                200\n",
       "teacher                     200\n",
       "nurse                       200\n",
       "software engineer           200\n",
       "painter                     200\n",
       "personal trainer            200\n",
       "dietitian                   200\n",
       "dentist                     200\n",
       "interior designer           200\n",
       "senior software engineer    200\n",
       "accountant                  200\n",
       "paralegal                   200\n",
       "Name: raw_title, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bios['raw_title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load Data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, val_test = train_test_split(bios,random_state=100, test_size=0.30, stratify=bios['raw_title'], shuffle=True)\n",
    "test, val = train_test_split(val_test,random_state=100, test_size=0.333, stratify=val_test['raw_title'], shuffle=True)\n",
    "train = train.reset_index()\n",
    "train.drop('index', axis=1, inplace=True)\n",
    "val = val.reset_index()\n",
    "val.drop('index', axis=1, inplace=True)\n",
    "test = test.reset_index()\n",
    "test.drop('index', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(560, 280, 1960)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test),len(val),len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.pkl', 'wb') as f:\n",
    "    pickle.dump(train, f)\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.pkl', 'wb') as f:\n",
    "    pickle.dump(val, f)\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.pkl', 'wb') as f:\n",
    "    pickle.dump(test, f)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pickle to Jason File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "uk_jobs = pd.DataFrame(dicts)\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>raw_title</th>\n",
       "      <th>gender</th>\n",
       "      <th>bio</th>\n",
       "      <th>counter_bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dr Ja Intarasoot is a Paediatric Dentist (Chil...</td>\n",
       "      <td>dentist</td>\n",
       "      <td>F</td>\n",
       "      <td>Dr Alice is a Paediatric _ (Children's _). She...</td>\n",
       "      <td>Dr Bob is a Paediatric _ (Children's _). he sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tabitha Beatty-Garcia is a Senior Accountant f...</td>\n",
       "      <td>accountant</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a Senior _ for Read King. In this rol...</td>\n",
       "      <td>Bob is a Senior _ for Read King. In this role,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Martin Binks is a clinical _ at Texas Tech Uni...</td>\n",
       "      <td>psychologist</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a clinical _ at Texas Tech University i...</td>\n",
       "      <td>Alice is a clinical _ at Texas Tech University...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Matt Harrison is a Senior Software Engineer wi...</td>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a _ with SpikeSource. When not busy pro...</td>\n",
       "      <td>Alice is a _ with SpikeSource. When not busy p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Thornton Abel is a lesser-known _ but still qu...</td>\n",
       "      <td>architect</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a lesser-known _ but still quite notabl...</td>\n",
       "      <td>Alice is a lesser-known _ but still quite nota...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955</th>\n",
       "      <td>Rachell Ekroos is a family _ practitioner spec...</td>\n",
       "      <td>nurse</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a family _ practitioner specializing ...</td>\n",
       "      <td>Bob is a family _ practitioner specializing in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1956</th>\n",
       "      <td>Swami Bodhananda Sarasvati is an accomplished ...</td>\n",
       "      <td>teacher</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is an accomplished _ of Vedanta, meditatio...</td>\n",
       "      <td>Alice is an accomplished _ of Vedanta, meditat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957</th>\n",
       "      <td>Renee Cox is a _ whose work display a deep con...</td>\n",
       "      <td>photographer</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a _ whose work display a deep concern...</td>\n",
       "      <td>Bob is a _ whose work display a deep concern f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1958</th>\n",
       "      <td>Katie Haneke is a Registered Dietitian working...</td>\n",
       "      <td>dietitian</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a Registered _ working at the Grand R...</td>\n",
       "      <td>Bob is a Registered _ working at the Grand Riv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1959</th>\n",
       "      <td>Dr. Tara McGee is a beautiful _ who owns a suc...</td>\n",
       "      <td>psychologist</td>\n",
       "      <td>F</td>\n",
       "      <td>Dr. Alice is a beautiful _ who owns a successf...</td>\n",
       "      <td>Dr. Bob is a beautiful _ who owns a successful...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1960 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    raw  \\\n",
       "0     Dr Ja Intarasoot is a Paediatric Dentist (Chil...   \n",
       "1     Tabitha Beatty-Garcia is a Senior Accountant f...   \n",
       "2     Martin Binks is a clinical _ at Texas Tech Uni...   \n",
       "3     Matt Harrison is a Senior Software Engineer wi...   \n",
       "4     Thornton Abel is a lesser-known _ but still qu...   \n",
       "...                                                 ...   \n",
       "1955  Rachell Ekroos is a family _ practitioner spec...   \n",
       "1956  Swami Bodhananda Sarasvati is an accomplished ...   \n",
       "1957  Renee Cox is a _ whose work display a deep con...   \n",
       "1958  Katie Haneke is a Registered Dietitian working...   \n",
       "1959  Dr. Tara McGee is a beautiful _ who owns a suc...   \n",
       "\n",
       "                     raw_title gender  \\\n",
       "0                      dentist      F   \n",
       "1                   accountant      F   \n",
       "2                 psychologist      M   \n",
       "3     senior software engineer      M   \n",
       "4                    architect      M   \n",
       "...                        ...    ...   \n",
       "1955                     nurse      F   \n",
       "1956                   teacher      M   \n",
       "1957              photographer      F   \n",
       "1958                 dietitian      F   \n",
       "1959              psychologist      F   \n",
       "\n",
       "                                                    bio  \\\n",
       "0     Dr Alice is a Paediatric _ (Children's _). She...   \n",
       "1     Alice is a Senior _ for Read King. In this rol...   \n",
       "2     Bob is a clinical _ at Texas Tech University i...   \n",
       "3     Bob is a _ with SpikeSource. When not busy pro...   \n",
       "4     Bob is a lesser-known _ but still quite notabl...   \n",
       "...                                                 ...   \n",
       "1955  Alice is a family _ practitioner specializing ...   \n",
       "1956  Bob is an accomplished _ of Vedanta, meditatio...   \n",
       "1957  Alice is a _ whose work display a deep concern...   \n",
       "1958  Alice is a Registered _ working at the Grand R...   \n",
       "1959  Dr. Alice is a beautiful _ who owns a successf...   \n",
       "\n",
       "                                            counter_bio  \n",
       "0     Dr Bob is a Paediatric _ (Children's _). he sp...  \n",
       "1     Bob is a Senior _ for Read King. In this role,...  \n",
       "2     Alice is a clinical _ at Texas Tech University...  \n",
       "3     Alice is a _ with SpikeSource. When not busy p...  \n",
       "4     Alice is a lesser-known _ but still quite nota...  \n",
       "...                                                 ...  \n",
       "1955  Bob is a family _ practitioner specializing in...  \n",
       "1956  Alice is an accomplished _ of Vedanta, meditat...  \n",
       "1957  Bob is a _ whose work display a deep concern f...  \n",
       "1958  Bob is a Registered _ working at the Grand Riv...  \n",
       "1959  Dr. Bob is a beautiful _ who owns a successful...  \n",
       "\n",
       "[1960 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>The Senior .net Developer will be part of the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nurse</td>\n",
       "      <td>Registered Nurse - Good CQC -  Paid breaks - ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nurse</td>\n",
       "      <td>It’s more than a possibility      Nurse - U...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dentist</td>\n",
       "      <td>We’re looking for an Associate Dentist to wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>software engineer</td>\n",
       "      <td>Role Overview     Looking for the next step...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2080</th>\n",
       "      <td>dentist</td>\n",
       "      <td>Description Associate Dentist required for ext...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2081</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>MediaMathhelps the world's top brands deliver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2082</th>\n",
       "      <td>photographer</td>\n",
       "      <td>Photographer    Altnagelvin, Londonderry BT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2083</th>\n",
       "      <td>personal trainer</td>\n",
       "      <td>Rare opportunity to join The Chapel - Luxury...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2084</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>Senior Software Engineer - Up to £75,000 - F...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2085 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         title  \\\n",
       "0     senior software engineer   \n",
       "1                        nurse   \n",
       "2                        nurse   \n",
       "3                      dentist   \n",
       "4            software engineer   \n",
       "...                        ...   \n",
       "2080                   dentist   \n",
       "2081  senior software engineer   \n",
       "2082              photographer   \n",
       "2083          personal trainer   \n",
       "2084  senior software engineer   \n",
       "\n",
       "                                            description  \n",
       "0     The Senior .net Developer will be part of the ...  \n",
       "1      Registered Nurse - Good CQC -  Paid breaks - ...  \n",
       "2        It’s more than a possibility      Nurse - U...  \n",
       "3      We’re looking for an Associate Dentist to wor...  \n",
       "4        Role Overview     Looking for the next step...  \n",
       "...                                                 ...  \n",
       "2080  Description Associate Dentist required for ext...  \n",
       "2081   MediaMathhelps the world's top brands deliver...  \n",
       "2082     Photographer    Altnagelvin, Londonderry BT...  \n",
       "2083    Rare opportunity to join The Chapel - Luxury...  \n",
       "2084    Senior Software Engineer - Up to £75,000 - F...  \n",
       "\n",
       "[2085 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uk_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "uk_jobs = pd.DataFrame(dicts)\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)\n",
    "\n",
    "home_path = '/share/hel/datasets/jobiqo/talent.com/JobRec/BM25/'\n",
    "\n",
    "uk_jobs['id'] = ['doc'+str(i+1) for i in range(len(uk_jobs))]\n",
    "\n",
    "#bios_true_gender = bios.drop(['raw','counter_bio'],axis='columns')\n",
    "#bios_true_gender.rename(columns = {'bio':'contents'}, inplace = True)\n",
    "#bios_true_gender.to_json(home_path + 'counterfactual_gender/bios.json', orient='records')\n",
    "\n",
    "\n",
    "uk_jobs.rename(columns = {'description':'contents'}, inplace = True)\n",
    "uk_jobs.to_json(home_path + 'uk_jobs.json', orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get TSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.pkl', 'rb') as f:\n",
    "    dicts = pickle.load(f)\n",
    "    data = pd.DataFrame(dicts)\n",
    "    data = data['bio']\n",
    "    data.to_csv('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.tsv', header=False, sep=\"\\t\") \n",
    "\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.pkl', 'rb') as f:\n",
    "    dicts = pickle.load(f)\n",
    "    data = pd.DataFrame(dicts)\n",
    "    data = data['bio']\n",
    "    data.to_csv('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.tsv', header=False, sep=\"\\t\")\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.pkl', 'rb') as f:\n",
    "    dicts = pickle.load(f)\n",
    "    data = pd.DataFrame(dicts)\n",
    "    data = data['bio']\n",
    "    data.to_csv('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.tsv', header=False, sep=\"\\t\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GEt BM25 hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance.\n",
      "2024-02-12 02:40:08,960 INFO  [main] index.IndexCollection (IndexCollection.java:391) - Setting log level to INFO\n",
      "2024-02-12 02:40:08,963 INFO  [main] index.IndexCollection (IndexCollection.java:394) - Starting indexer...\n",
      "2024-02-12 02:40:08,963 INFO  [main] index.IndexCollection (IndexCollection.java:395) - ============ Loading Parameters ============\n",
      "2024-02-12 02:40:08,963 INFO  [main] index.IndexCollection (IndexCollection.java:396) - DocumentCollection path: /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:397) - CollectionClass: JsonCollection\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:398) - Generator: DefaultLuceneDocumentGenerator\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:399) - Threads: 1\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:400) - Language: en\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:401) - Stemmer: porter\n",
      "2024-02-12 02:40:08,964 INFO  [main] index.IndexCollection (IndexCollection.java:402) - Keep stopwords? false\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:403) - Stopwords: null\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:404) - Store positions? true\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:405) - Store docvectors? true\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:406) - Store document \"contents\" field? false\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:407) - Store document \"raw\" field? true\n",
      "2024-02-12 02:40:08,965 INFO  [main] index.IndexCollection (IndexCollection.java:408) - Additional fields to index: []\n",
      "2024-02-12 02:40:08,966 INFO  [main] index.IndexCollection (IndexCollection.java:409) - Optimize (merge segments)? false\n",
      "2024-02-12 02:40:08,966 INFO  [main] index.IndexCollection (IndexCollection.java:410) - Whitelist: null\n",
      "2024-02-12 02:40:08,967 INFO  [main] index.IndexCollection (IndexCollection.java:411) - Pretokenized?: false\n",
      "2024-02-12 02:40:08,967 INFO  [main] index.IndexCollection (IndexCollection.java:412) - Index path: /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/indexes\n",
      "2024-02-12 02:40:08,971 INFO  [main] index.IndexCollection (IndexCollection.java:450) - ============ Indexing Collection ============\n",
      "2024-02-12 02:40:09,361 INFO  [main] index.IndexCollection (IndexCollection.java:565) - Thread pool with 1 threads initialized.\n",
      "2024-02-12 02:40:09,361 INFO  [main] index.IndexCollection (IndexCollection.java:567) - Initializing collection in /share/hel/datasets/jobiqo/talent.com/JobRec/BM25\n",
      "2024-02-12 02:40:09,368 INFO  [main] index.IndexCollection (IndexCollection.java:576) - 1 file found\n",
      "2024-02-12 02:40:09,369 INFO  [main] index.IndexCollection (IndexCollection.java:577) - Starting to index...\n",
      "2024-02-12 02:40:15,578 DEBUG [pool-2-thread-1] index.IndexCollection$LocalIndexerThread (IndexCollection.java:356) - BM25/uk_jobs.json: 2085 docs added.\n",
      "2024-02-12 02:40:19,351 INFO  [main] index.IndexCollection (IndexCollection.java:633) - Indexing Complete! 2,085 documents indexed\n",
      "2024-02-12 02:40:19,351 INFO  [main] index.IndexCollection (IndexCollection.java:634) - ============ Final Counter Values ============\n",
      "2024-02-12 02:40:19,352 INFO  [main] index.IndexCollection (IndexCollection.java:635) - indexed:            2,085\n",
      "2024-02-12 02:40:19,352 INFO  [main] index.IndexCollection (IndexCollection.java:636) - unindexable:            0\n",
      "2024-02-12 02:40:19,352 INFO  [main] index.IndexCollection (IndexCollection.java:637) - empty:                  0\n",
      "2024-02-12 02:40:19,352 INFO  [main] index.IndexCollection (IndexCollection.java:638) - skipped:                0\n",
      "2024-02-12 02:40:19,352 INFO  [main] index.IndexCollection (IndexCollection.java:639) - errors:                 0\n",
      "2024-02-12 02:40:19,365 INFO  [main] index.IndexCollection (IndexCollection.java:642) - Total 2,085 documents indexed in 00:00:10\n"
     ]
    }
   ],
   "source": [
    "!python -m pyserini.index.lucene \\\n",
    "  --collection JsonCollection \\\n",
    "  --input /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/ \\\n",
    "  --index /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/indexes \\\n",
    "  --generator DefaultLuceneDocumentGenerator \\\n",
    "  --threads 1 \\\n",
    "  --storePositions --storeDocvectors --storeRaw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.tsv topics, saving to /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_train_hits.txt...\n",
      "100%|███████████████████████████████████████| 1960/1960 [02:05<00:00, 15.65it/s]\n"
     ]
    }
   ],
   "source": [
    "!python -m pyserini.search.lucene  --index /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/indexes --topics /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.tsv --output /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_train_hits.txt  --bm25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.tsv topics, saving to /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_val_hits.txt...\n",
      "100%|█████████████████████████████████████████| 280/280 [00:18<00:00, 15.18it/s]\n"
     ]
    }
   ],
   "source": [
    "!python -m pyserini.search.lucene  --index /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/indexes --topics /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.tsv --output /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_val_hits.txt  --bm25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.tsv topics, saving to /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_test_hits.txt...\n",
      "100%|█████████████████████████████████████████| 560/560 [00:36<00:00, 15.47it/s]\n"
     ]
    }
   ],
   "source": [
    "!python -m pyserini.search.lucene  --index /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/indexes --topics /share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.tsv --output /share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_test_hits.txt  --bm25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Neutrality Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "senior software engineer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deepak/anaconda3/envs/evonlp/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nurse\n",
      "dentist\n",
      "software engineer\n",
      "accountant\n",
      "dietitian\n",
      "architect\n",
      "personal trainer\n",
      "teacher\n",
      "painter\n",
      "interior designer\n",
      "psychologist\n",
      "paralegal\n",
      "photographer\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import regex as re\n",
    "from nltk.corpus import wordnet as wn\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np\n",
    "\n",
    "cachedStopWords = stopwords.words(\"english\")\n",
    "\n",
    "def cleaner(s):\n",
    "    s = s.replace('[SEP]','')\n",
    "    return re.sub(r'[^A-Za-z0-9 ]+', '', s).strip()\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)\n",
    "\n",
    "dictionary_jobs = {}\n",
    "for job in uk_jobs['title'].unique():\n",
    "#for job in ['senior software engineer']:\n",
    "    print(job)\n",
    "    with open( '/home/deepak/RecSys2023/IG_bert/'+job+'.pkl', 'rb') as file:\n",
    "        dictionary_dictionary = pickle.load(file)\n",
    "        dictionary_words = {}    \n",
    "        for key in dictionary_dictionary.keys():\n",
    "            for word in dictionary_dictionary[key]:\n",
    "                if word not in cachedStopWords and not word.isdigit() and len(cleaner(word))>2 and wn.synsets(word)!=[] :\n",
    "                    if word in dictionary_words.keys():\n",
    "                        dictionary_words[cleaner(word)].append(dictionary_dictionary[key][word])\n",
    "                    else:\n",
    "                        dictionary_words[cleaner(word)] = [dictionary_dictionary[key][word].cpu().detach().numpy()]\n",
    "\n",
    "        for key in dictionary_words.keys():\n",
    "            dictionary_words[key] =  float(sum(dictionary_words[key])/len(dictionary_words[key])) # denominator non zero\n",
    "\n",
    "        dictionary_jobs[job] = {key:testue for key,testue in dictionary_words.items() if testue >sorted(dictionary_words.values(),reverse=True)[20]}\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/dictionary_jobs.pkl', 'wb') as f:\n",
    "    pickle.dump(dictionary_jobs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "uk_jobs = pd.DataFrame(dicts)\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>The Senior .net Developer will be part of the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nurse</td>\n",
       "      <td>Registered Nurse - Good CQC -  Paid breaks - ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nurse</td>\n",
       "      <td>It’s more than a possibility      Nurse - U...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dentist</td>\n",
       "      <td>We’re looking for an Associate Dentist to wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>software engineer</td>\n",
       "      <td>Role Overview     Looking for the next step...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2080</th>\n",
       "      <td>dentist</td>\n",
       "      <td>Description Associate Dentist required for ext...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2081</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>MediaMathhelps the world's top brands deliver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2082</th>\n",
       "      <td>photographer</td>\n",
       "      <td>Photographer    Altnagelvin, Londonderry BT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2083</th>\n",
       "      <td>personal trainer</td>\n",
       "      <td>Rare opportunity to join The Chapel - Luxury...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2084</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>Senior Software Engineer - Up to £75,000 - F...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2085 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         title  \\\n",
       "0     senior software engineer   \n",
       "1                        nurse   \n",
       "2                        nurse   \n",
       "3                      dentist   \n",
       "4            software engineer   \n",
       "...                        ...   \n",
       "2080                   dentist   \n",
       "2081  senior software engineer   \n",
       "2082              photographer   \n",
       "2083          personal trainer   \n",
       "2084  senior software engineer   \n",
       "\n",
       "                                            description  \n",
       "0     The Senior .net Developer will be part of the ...  \n",
       "1      Registered Nurse - Good CQC -  Paid breaks - ...  \n",
       "2        It’s more than a possibility      Nurse - U...  \n",
       "3      We’re looking for an Associate Dentist to wor...  \n",
       "4        Role Overview     Looking for the next step...  \n",
       "...                                                 ...  \n",
       "2080  Description Associate Dentist required for ext...  \n",
       "2081   MediaMathhelps the world's top brands deliver...  \n",
       "2082     Photographer    Altnagelvin, Londonderry BT...  \n",
       "2083    Rare opportunity to join The Chapel - Luxury...  \n",
       "2084    Senior Software Engineer - Up to £75,000 - F...  \n",
       "\n",
       "[2085 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uk_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "\n",
    "class DocNeutrality:\n",
    "    def __init__(self, representative_words_path='/share/hel/datasets/jobiqo/talent.com/JobRec/dictionary_jobs.pkl',threshold=1):\n",
    "        self.representative_words_path = representative_words_path\n",
    "        self.threshold = threshold\n",
    "\n",
    "        with open(self.representative_words_path, 'rb') as f:\n",
    "            self.dictionary_jobs = pickle.load(f)\n",
    "\n",
    "\n",
    "    def doc_neutrality(self, doc: str, job: str):\n",
    "        bias = 0\n",
    "        frequency = 0\n",
    "        word_dict = self.dictionary_jobs[job]\n",
    "        for word in word_dict.keys():\n",
    "            bias = bias + (doc.count(word))*word_dict[word]\n",
    "            frequency = frequency +doc.count(word)\n",
    "        neutrality = 1-(bias/frequency) if frequency>self.threshold else 1\n",
    "        \n",
    "        return neutrality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>neutrality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>The Senior .net Developer will be part of the ...</td>\n",
       "      <td>0.940670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nurse</td>\n",
       "      <td>Registered Nurse - Good CQC -  Paid breaks - ...</td>\n",
       "      <td>0.961643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nurse</td>\n",
       "      <td>It’s more than a possibility      Nurse - U...</td>\n",
       "      <td>0.960256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dentist</td>\n",
       "      <td>We’re looking for an Associate Dentist to wor...</td>\n",
       "      <td>0.953347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>software engineer</td>\n",
       "      <td>Role Overview     Looking for the next step...</td>\n",
       "      <td>0.938174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2080</th>\n",
       "      <td>dentist</td>\n",
       "      <td>Description Associate Dentist required for ext...</td>\n",
       "      <td>0.957394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2081</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>MediaMathhelps the world's top brands deliver...</td>\n",
       "      <td>0.942938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2082</th>\n",
       "      <td>photographer</td>\n",
       "      <td>Photographer    Altnagelvin, Londonderry BT...</td>\n",
       "      <td>0.956828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2083</th>\n",
       "      <td>personal trainer</td>\n",
       "      <td>Rare opportunity to join The Chapel - Luxury...</td>\n",
       "      <td>0.962939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2084</th>\n",
       "      <td>senior software engineer</td>\n",
       "      <td>Senior Software Engineer - Up to £75,000 - F...</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2085 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         title  \\\n",
       "0     senior software engineer   \n",
       "1                        nurse   \n",
       "2                        nurse   \n",
       "3                      dentist   \n",
       "4            software engineer   \n",
       "...                        ...   \n",
       "2080                   dentist   \n",
       "2081  senior software engineer   \n",
       "2082              photographer   \n",
       "2083          personal trainer   \n",
       "2084  senior software engineer   \n",
       "\n",
       "                                            description  neutrality  \n",
       "0     The Senior .net Developer will be part of the ...    0.940670  \n",
       "1      Registered Nurse - Good CQC -  Paid breaks - ...    0.961643  \n",
       "2        It’s more than a possibility      Nurse - U...    0.960256  \n",
       "3      We’re looking for an Associate Dentist to wor...    0.953347  \n",
       "4        Role Overview     Looking for the next step...    0.938174  \n",
       "...                                                 ...         ...  \n",
       "2080  Description Associate Dentist required for ext...    0.957394  \n",
       "2081   MediaMathhelps the world's top brands deliver...    0.942938  \n",
       "2082     Photographer    Altnagelvin, Londonderry BT...    0.956828  \n",
       "2083    Rare opportunity to join The Chapel - Luxury...    0.962939  \n",
       "2084    Senior Software Engineer - Up to £75,000 - F...    1.000000  \n",
       "\n",
       "[2085 rows x 3 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uk_jobs['neutrality'] = [DocNeutrality().doc_neutrality(doc=uk_jobs['description'][i],job=uk_jobs['title'][i]) for i in range(len(uk_jobs))]\n",
    "\n",
    "uk_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'wb') as f:\n",
    "    pickle.dump(uk_jobs, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train and validation set ready to load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/deepak/ipykernel_3763/1002600804.py:3: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "from tqdm.autonotebook import tqdm\n",
    "import random\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)\n",
    "\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "bios_train = pd.DataFrame(dicts)\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_val.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "bios_val = pd.DataFrame(dicts)\n",
    "\n",
    "\n",
    "val_hits = pd.read_csv('/share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_val_hits.txt', sep=' ', header=None, names=['query_id', 'Q0', 'doc_id', 'rank', 'score','Anserini'])\n",
    "train_hits = pd.read_csv('/share/hel/datasets/jobiqo/talent.com/JobRec/BM25/unbalanced_train_hits.txt', sep=' ', header=None, names=['query_id', 'Q0', 'doc_id', 'rank', 'score','Anserini'])\n",
    "val_hits = val_hits.drop(['Q0','rank','score','Anserini'], axis='columns')\n",
    "val_hits['doc_id'] = val_hits['doc_id'].replace('doc','',regex=True).astype(int)\n",
    "train_hits = train_hits.drop(['Q0','rank','score','Anserini'], axis='columns')\n",
    "train_hits['doc_id'] = train_hits['doc_id'].replace('doc','',regex=True).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1960/1960 [00:20<00:00, 93.79it/s] \n"
     ]
    }
   ],
   "source": [
    "\n",
    "#In Train for every datapoint with label 1 we add 4 negative samples\n",
    "from sentence_transformers import InputExample\n",
    "\n",
    "train_samples = []\n",
    "\n",
    "for id in tqdm(range(len(bios_train))):\n",
    "\n",
    "    new_uk_jobs = uk_jobs.loc[train_hits[train_hits['query_id']==id]['doc_id']-1]\n",
    "\n",
    "    query = bios_train['bio'][id]\n",
    "    job = bios_train['raw_title'][id]\n",
    "    \n",
    "    pos_ad= new_uk_jobs.loc[(new_uk_jobs.title == job)][['description','neutrality']].sample(n=1)\n",
    "    train_samples.append(InputExample(texts=[query, pos_ad['description'].values[0]], label=1, gender=pos_ad['neutrality'].values[0]))\n",
    "    for looper in range(4):\n",
    "        neg_passage = new_uk_jobs.loc[new_uk_jobs.title != job][['description','neutrality']].sample(n=1)\n",
    "        train_samples.append(InputExample(texts=[query, neg_passage['description'].values[0]],label=0, gender=neg_passage['neutrality'].values[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9800\n"
     ]
    }
   ],
   "source": [
    "print(len(train_samples))\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_train_samples.pkl', 'wb') as f:\n",
    "    pickle.dump(train_samples, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 280/280 [00:01<00:00, 250.92it/s]\n"
     ]
    }
   ],
   "source": [
    "# MAE loss fairness, bceloss task\n",
    "import numpy as np \n",
    "\n",
    "dev_samples = {}\n",
    "\n",
    "for id in tqdm(range(len(bios_val))):\n",
    "\n",
    "    new_uk_jobs = uk_jobs.loc[val_hits[val_hits['query_id']==id]['doc_id']-1]\n",
    "\n",
    "    query = bios_val['bio'][id]\n",
    "    job = bios_val['raw_title'][id]\n",
    "\n",
    "    dev_samples[id] = {'query': query, 'positive': set(), 'negative': set(), 'neutrality': list()}\n",
    "\n",
    "    pos_passage = new_uk_jobs.loc[(new_uk_jobs.title == job)][['description','neutrality']].sample(n=1)\n",
    "\n",
    "    neg_passage = new_uk_jobs.loc[(new_uk_jobs.title != job)][['description','neutrality']].sample(n=1)\n",
    "\n",
    "    dev_samples[id]['positive'].add(pos_passage['description'].values[0])\n",
    "    dev_samples[id]['neutrality'].append(pos_passage['neutrality'].values[0])\n",
    "    dev_samples[id]['negative'].add(neg_passage['description'].values[0])\n",
    "    dev_samples[id]['neutrality'].append(neg_passage['neutrality'].values[0])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280\n"
     ]
    }
   ],
   "source": [
    "print(len(dev_samples))\n",
    "\n",
    "with open('/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_dev_samples.pkl', 'wb') as f:\n",
    "    pickle.dump(dev_samples, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1500\n",
      "1390\n",
      "5915\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "print(random.randint(0,10000))\n",
    "print(random.randint(0,10000))\n",
    "print(random.randint(0,10000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 ('adapterfusion': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "54416a898087e692e51f3d654c363a8a05558aed57ab1c18f3e4f84290aeecd3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}