03-filter_nodes_edges.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "03-filter_nodes_edges.ipynb\n",
    "- Remove edges with nodes with no umls type or label\n",
    "- Remove the following predicates: ['compared_with', 'higher_than', 'lower_than', 'different_from', 'different_than', \n",
    "'same_as','OCCURS_IN', 'PROCESS_OF', 'DIAGNOSES', 'METHOD_OF', 'USES','AUGMENTS', 'ADMINISTERED_TO', 'COMPLICATES']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: TkAgg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "%pylab\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import re\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/numpy/lib/arraysetops.py:472: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUBJECT_CUI</th>\n",
       "      <th>PREDICATE</th>\n",
       "      <th>OBJECT_CUI</th>\n",
       "      <th>PMID</th>\n",
       "      <th>NEG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>C1412045</td>\n",
       "      <td>AFFECTS</td>\n",
       "      <td>C0020291</td>\n",
       "      <td>6298464</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>C1412045</td>\n",
       "      <td>AFFECTS</td>\n",
       "      <td>C0023946</td>\n",
       "      <td>20801151</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>C1412045</td>\n",
       "      <td>AFFECTS</td>\n",
       "      <td>C0028754</td>\n",
       "      <td>19789049</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>C1412045</td>\n",
       "      <td>AFFECTS</td>\n",
       "      <td>C0597304</td>\n",
       "      <td>1409557</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>C1412045</td>\n",
       "      <td>AFFECTS</td>\n",
       "      <td>C0599816</td>\n",
       "      <td>7617239</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    SUBJECT_CUI PREDICATE OBJECT_CUI      PMID    NEG\n",
       "379    C1412045   AFFECTS   C0020291   6298464  False\n",
       "380    C1412045   AFFECTS   C0023946  20801151  False\n",
       "381    C1412045   AFFECTS   C0028754  19789049  False\n",
       "382    C1412045   AFFECTS   C0597304   1409557  False\n",
       "383    C1412045   AFFECTS   C0599816   7617239  False"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edges = pd.read_csv('edges3.csv', index_col=0)\n",
    "edges.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>LABEL</th>\n",
       "      <th>umls_type</th>\n",
       "      <th>umls_type_label</th>\n",
       "      <th>blm_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0061133</td>\n",
       "      <td>gastrin releasing peptide (14-27)</td>\n",
       "      <td>T116</td>\n",
       "      <td>Amino Acid, Peptide, or Protein</td>\n",
       "      <td>protein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C1523610</td>\n",
       "      <td>regulation of tube length, open tracheal system</td>\n",
       "      <td>T042</td>\n",
       "      <td>Organ or Tissue Function</td>\n",
       "      <td>biological_process_or_activity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0312636</td>\n",
       "      <td>Antibody to hepatitis E virus</td>\n",
       "      <td>T116|T129</td>\n",
       "      <td>Amino Acid, Peptide, or Protein|Immunologic Fa...</td>\n",
       "      <td>biological_entity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0539817</td>\n",
       "      <td>cytochrome p30</td>\n",
       "      <td>T116|T126</td>\n",
       "      <td>Amino Acid, Peptide, or Protein|Enzyme</td>\n",
       "      <td>protein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C0406240</td>\n",
       "      <td>Photosensitive atopic dermatitis</td>\n",
       "      <td>T047</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>disease_or_phenotypic_feature</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID                                            LABEL  umls_type  \\\n",
       "0  C0061133                gastrin releasing peptide (14-27)       T116   \n",
       "1  C1523610  regulation of tube length, open tracheal system       T042   \n",
       "2  C0312636                    Antibody to hepatitis E virus  T116|T129   \n",
       "4  C0539817                                   cytochrome p30  T116|T126   \n",
       "6  C0406240                 Photosensitive atopic dermatitis       T047   \n",
       "\n",
       "                                     umls_type_label  \\\n",
       "0                    Amino Acid, Peptide, or Protein   \n",
       "1                           Organ or Tissue Function   \n",
       "2  Amino Acid, Peptide, or Protein|Immunologic Fa...   \n",
       "4             Amino Acid, Peptide, or Protein|Enzyme   \n",
       "6                                Disease or Syndrome   \n",
       "\n",
       "                         blm_type  \n",
       "0                         protein  \n",
       "1  biological_process_or_activity  \n",
       "2               biological_entity  \n",
       "4                         protein  \n",
       "6   disease_or_phenotypic_feature  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes = pd.read_csv(\"nodes_blm.csv\", index_col=0)\n",
    "nodes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180491\n",
      "22280924\n"
     ]
    }
   ],
   "source": [
    "print(len(nodes))\n",
    "print(len(edges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22280924\n",
      "15378937\n"
     ]
    }
   ],
   "source": [
    "# remove edges with no nodes\n",
    "print(len(edges))\n",
    "edges = edges[edges.SUBJECT_CUI.isin(nodes.ID) & edges.OBJECT_CUI.isin(nodes.ID)]\n",
    "print(len(edges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LOCATION_OF         1863675\n",
       "INTERACTS_WITH      1850601\n",
       "COEXISTS_WITH       1707696\n",
       "AFFECTS             1443544\n",
       "PART_OF             1231194\n",
       "STIMULATES          1159243\n",
       "INHIBITS             925338\n",
       "ASSOCIATED_WITH      813132\n",
       "CAUSES               716141\n",
       "AUGMENTS             652698\n",
       "TREATS               612205\n",
       "PRODUCES             553511\n",
       "DISRUPTS             471637\n",
       "compared_with        351305\n",
       "ISA                  247356\n",
       "PREDISPOSES          230306\n",
       "PREVENTS             117892\n",
       "higher_than          107865\n",
       "CONVERTS_TO           61644\n",
       "PROCESS_OF            54726\n",
       "MANIFESTATION_OF      36462\n",
       "COMPLICATES           33992\n",
       "USES                  25959\n",
       "PRECEDES              24652\n",
       "lower_than            21266\n",
       "same_as               21083\n",
       "OCCURS_IN             16317\n",
       "ADMINISTERED_TO       13971\n",
       "different_from         7959\n",
       "DIAGNOSES              5378\n",
       "METHOD_OF               189\n",
       "Name: PREDICATE, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vc = edges.PREDICATE.value_counts()\n",
    "vc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_edges(df, drop_edges):\n",
    "    idx = df.query('PREDICATE in @drop_edges').index\n",
    "    df.drop(idx, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15378937\n",
      "14066229\n"
     ]
    }
   ],
   "source": [
    "# remove edges we don't need\n",
    "print(len(edges))\n",
    "remove_edges = ['compared_with', 'higher_than', 'lower_than', 'different_from', 'different_than', 'same_as',\n",
    "               'OCCURS_IN', 'PROCESS_OF', 'DIAGNOSES', 'METHOD_OF', 'USES',\n",
    "               'AUGMENTS', 'ADMINISTERED_TO', 'COMPLICATES']\n",
    "drop_edges(edges, remove_edges)\n",
    "print(len(edges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180491\n",
      "165765\n"
     ]
    }
   ],
   "source": [
    "# remove nodes with no edges\n",
    "print(len(nodes))\n",
    "nodes = nodes[nodes.ID.isin(set(list(edges.SUBJECT_CUI) + list(edges.OBJECT_CUI)))]\n",
    "print(len(nodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LOCATION_OF         1863675\n",
       "INTERACTS_WITH      1850601\n",
       "COEXISTS_WITH       1707696\n",
       "AFFECTS             1443544\n",
       "PART_OF             1231194\n",
       "STIMULATES          1159243\n",
       "INHIBITS             925338\n",
       "ASSOCIATED_WITH      813132\n",
       "CAUSES               716141\n",
       "TREATS               612205\n",
       "PRODUCES             553511\n",
       "DISRUPTS             471637\n",
       "ISA                  247356\n",
       "PREDISPOSES          230306\n",
       "PREVENTS             117892\n",
       "CONVERTS_TO           61644\n",
       "MANIFESTATION_OF      36462\n",
       "PRECEDES              24652\n",
       "Name: PREDICATE, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vc = edges.PREDICATE.value_counts()\n",
    "vc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "edges.to_csv('edges_filtered.csv', index=False)\n",
    "nodes.to_csv(\"nodes_filtered.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}