Superscan v 0.0.1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Covid-19 Chest X-Ray Image Scanner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the data for positive samples\n",
    "FILE_PATH = 'data/metadata.csv'\n",
    "IMAGES_PATH = 'data/images'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(422, 29)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(FILE_PATH)\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientid</th>\n",
       "      <th>offset</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>finding</th>\n",
       "      <th>survival</th>\n",
       "      <th>intubated</th>\n",
       "      <th>intubation_present</th>\n",
       "      <th>went_icu</th>\n",
       "      <th>in_icu</th>\n",
       "      <th>...</th>\n",
       "      <th>date</th>\n",
       "      <th>location</th>\n",
       "      <th>folder</th>\n",
       "      <th>filename</th>\n",
       "      <th>doi</th>\n",
       "      <th>url</th>\n",
       "      <th>license</th>\n",
       "      <th>clinical_notes</th>\n",
       "      <th>other_notes</th>\n",
       "      <th>Unnamed: 28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>M</td>\n",
       "      <td>65.0</td>\n",
       "      <td>COVID-19</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>January 22, 2020</td>\n",
       "      <td>Cho Ray Hospital, Ho Chi Minh City, Vietnam</td>\n",
       "      <td>images</td>\n",
       "      <td>auntminnie-a-2020_01_28_23_51_6665_2020_01_28_...</td>\n",
       "      <td>10.1056/nejmc2001272</td>\n",
       "      <td>https://www.nejm.org/doi/full/10.1056/NEJMc200...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>On January 22, 2020, a 65-year-old man with a ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>M</td>\n",
       "      <td>65.0</td>\n",
       "      <td>COVID-19</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>January 25, 2020</td>\n",
       "      <td>Cho Ray Hospital, Ho Chi Minh City, Vietnam</td>\n",
       "      <td>images</td>\n",
       "      <td>auntminnie-b-2020_01_28_23_51_6665_2020_01_28_...</td>\n",
       "      <td>10.1056/nejmc2001272</td>\n",
       "      <td>https://www.nejm.org/doi/full/10.1056/NEJMc200...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>On January 22, 2020, a 65-year-old man with a ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>M</td>\n",
       "      <td>65.0</td>\n",
       "      <td>COVID-19</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>January 27, 2020</td>\n",
       "      <td>Cho Ray Hospital, Ho Chi Minh City, Vietnam</td>\n",
       "      <td>images</td>\n",
       "      <td>auntminnie-c-2020_01_28_23_51_6665_2020_01_28_...</td>\n",
       "      <td>10.1056/nejmc2001272</td>\n",
       "      <td>https://www.nejm.org/doi/full/10.1056/NEJMc200...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>On January 22, 2020, a 65-year-old man with a ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>6.0</td>\n",
       "      <td>M</td>\n",
       "      <td>65.0</td>\n",
       "      <td>COVID-19</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>January 28, 2020</td>\n",
       "      <td>Cho Ray Hospital, Ho Chi Minh City, Vietnam</td>\n",
       "      <td>images</td>\n",
       "      <td>auntminnie-d-2020_01_28_23_51_6665_2020_01_28_...</td>\n",
       "      <td>10.1056/nejmc2001272</td>\n",
       "      <td>https://www.nejm.org/doi/full/10.1056/NEJMc200...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>On January 22, 2020, a 65-year-old man with a ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>F</td>\n",
       "      <td>52.0</td>\n",
       "      <td>COVID-19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>January 25, 2020</td>\n",
       "      <td>Changhua Christian Hospital, Changhua City, Ta...</td>\n",
       "      <td>images</td>\n",
       "      <td>nejmc2001573_f1a.jpeg</td>\n",
       "      <td>10.1056/NEJMc2001573</td>\n",
       "      <td>https://www.nejm.org/doi/full/10.1056/NEJMc200...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>diffuse infiltrates in the bilateral lower lungs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   patientid  offset sex   age   finding survival intubated  \\\n",
       "0          2     0.0   M  65.0  COVID-19        Y       NaN   \n",
       "1          2     3.0   M  65.0  COVID-19        Y       NaN   \n",
       "2          2     5.0   M  65.0  COVID-19        Y       NaN   \n",
       "3          2     6.0   M  65.0  COVID-19        Y       NaN   \n",
       "4          4     0.0   F  52.0  COVID-19      NaN       NaN   \n",
       "\n",
       "  intubation_present went_icu in_icu  ...              date  \\\n",
       "0                NaN      NaN    NaN  ...  January 22, 2020   \n",
       "1                NaN      NaN    NaN  ...  January 25, 2020   \n",
       "2                NaN      NaN    NaN  ...  January 27, 2020   \n",
       "3                NaN      NaN    NaN  ...  January 28, 2020   \n",
       "4                NaN      NaN    NaN  ...  January 25, 2020   \n",
       "\n",
       "                                            location  folder  \\\n",
       "0        Cho Ray Hospital, Ho Chi Minh City, Vietnam  images   \n",
       "1        Cho Ray Hospital, Ho Chi Minh City, Vietnam  images   \n",
       "2        Cho Ray Hospital, Ho Chi Minh City, Vietnam  images   \n",
       "3        Cho Ray Hospital, Ho Chi Minh City, Vietnam  images   \n",
       "4  Changhua Christian Hospital, Changhua City, Ta...  images   \n",
       "\n",
       "                                            filename                   doi  \\\n",
       "0  auntminnie-a-2020_01_28_23_51_6665_2020_01_28_...  10.1056/nejmc2001272   \n",
       "1  auntminnie-b-2020_01_28_23_51_6665_2020_01_28_...  10.1056/nejmc2001272   \n",
       "2  auntminnie-c-2020_01_28_23_51_6665_2020_01_28_...  10.1056/nejmc2001272   \n",
       "3  auntminnie-d-2020_01_28_23_51_6665_2020_01_28_...  10.1056/nejmc2001272   \n",
       "4                              nejmc2001573_f1a.jpeg  10.1056/NEJMc2001573   \n",
       "\n",
       "                                                 url  license  \\\n",
       "0  https://www.nejm.org/doi/full/10.1056/NEJMc200...      NaN   \n",
       "1  https://www.nejm.org/doi/full/10.1056/NEJMc200...      NaN   \n",
       "2  https://www.nejm.org/doi/full/10.1056/NEJMc200...      NaN   \n",
       "3  https://www.nejm.org/doi/full/10.1056/NEJMc200...      NaN   \n",
       "4  https://www.nejm.org/doi/full/10.1056/NEJMc200...      NaN   \n",
       "\n",
       "                                      clinical_notes other_notes Unnamed: 28  \n",
       "0  On January 22, 2020, a 65-year-old man with a ...         NaN         NaN  \n",
       "1  On January 22, 2020, a 65-year-old man with a ...         NaN         NaN  \n",
       "2  On January 22, 2020, a 65-year-old man with a ...         NaN         NaN  \n",
       "3  On January 22, 2020, a 65-year-old man with a ...         NaN         NaN  \n",
       "4   diffuse infiltrates in the bilateral lower lungs         NaN         NaN  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "TARGET_DIR = 'data/covid'\n",
    "\n",
    "if not os.path.exists(TARGET_DIR):\n",
    "    os.mkdir(TARGET_DIR)\n",
    "    print('Covid folder created!')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Move covid positive images to their own folder \n",
    "counter = 0\n",
    "\n",
    "for (i, row) in data.iterrows():\n",
    "    if row['finding'] == 'COVID-19' and row['view'] == 'PA':\n",
    "        try:\n",
    "            filename = row['filename']\n",
    "            image_path = os.path.join(IMAGES_PATH,filename)\n",
    "            image_copy_path = os.path.join(TARGET_DIR,filename)\n",
    "            shutil.copy2(image_path,image_copy_path)\n",
    "            # print('Moving image ', counter)\n",
    "            counter += 1\n",
    "        except:\n",
    "            print('Filename not found...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample normal image from Kaggle pneumonia dataset\n",
    "import random \n",
    "KAGGLE_FILE_PATH = 'kaggle/train/NORMAL'\n",
    "TARGET_NORMAL_DIR = 'data/normal'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_names = os.listdir(KAGGLE_FILE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'random' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-27b274a477ef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'random' is not defined"
     ]
    }
   ],
   "source": [
    "random.shuffle(image_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for in range(142):\n",
    "    image_name = image_names[i]\n",
    "    image_path = os.path.join(KAGGLE_FILE_PATH, image_name)\n",
    "    target_path = os.path.join(TARGET_NORMAL_DIR, image_name)\n",
    "    \n",
    "    shutil.copy2(image_path, target_path)\n",
    "    \n",
    "    print('Copying image ', i)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}