GradProject_NB3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<h1> DS200A Computer Vision Assignment</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>  Part Three: Training Models </h2>\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import skimage\n",
    "import sklearn\n",
    "from sklearn import preprocessing\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_hdf('feature.h5', 'feature')\n",
    "TRAIN_RATIO = 0.9\n",
    "\n",
    "# Split the data into a training set, and test set \n",
    "def train_test_split(df):\n",
    "    df = sklearn.utils.shuffle(df, random_state=29)\n",
    "    train_n = int(len(df) * TRAIN_RATIO)\n",
    "    return df[:train_n], df[train_n:]\n",
    "\n",
    "# Calculate the accuracy percentage of the predicted values\n",
    "def accuracy(pred, actual):\n",
    "    return (pred == actual).mean()\n",
    "\n",
    "def get_X_y(df):\n",
    "    \"\"\"\n",
    "    for a DataFrame, split it for feature data X and label y\n",
    "    \"\"\"\n",
    "    X = df.copy()\n",
    "    X = X.drop(\"Encoding\", axis=1).as_matrix()\n",
    "    y = df[\"Encoding\"].values\n",
    "    return X, y\n",
    "\n",
    "def feature_normalize(X):\n",
    "    \"\"\"\n",
    "    normalized features to mean 0 and variance 1.\n",
    "    \"\"\"\n",
    "    min_max_scaler = preprocessing.MinMaxScaler()\n",
    "    X_scaled = min_max_scaler.fit_transform(X)\n",
    "    return X_scaled\n",
    "\n",
    "# load data\n",
    "train_set, val_set = train_test_split(df)\n",
    "\n",
    "X_train, y_train = get_X_y(train_set)\n",
    "X_val, y_val = get_X_y(val_set)\n",
    "X_train = feature_normalize(X_train)\n",
    "X_val = feature_normalize(X_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>  Train models using all of the following methods below. Be sure to drop the actual image column, and the encoding</h3>\t\n",
    "Take note of the differences in accuracy, and methods.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# support function for evaluate model and print accuracy\n",
    "def print_model_accuracy(model, train_acc, val_acc):\n",
    "    # print parameters of model\n",
    "    print(model)\n",
    "    \n",
    "    # train the model\n",
    "    m = model.fit(X_train, y_train)\n",
    "    \n",
    "    # get prediction\n",
    "    yhat = m.predict(X_train)\n",
    "    \n",
    "    # calculate, store and print accuracy on training \n",
    "    # set and validation set\n",
    "    train_acc.append(accuracy(yhat, y_train))\n",
    "    print(\"Accuracy on train = {}\".format(accuracy(yhat, y_train)))\n",
    "    yhat = m.predict(X_val)\n",
    "    val_acc.append(accuracy(yhat, y_val))\n",
    "    print(\"Accuracy on val = {}\".format(accuracy(yhat, y_val)))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# search for different parameters\n",
    "train_list = []\n",
    "val_list = []\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', C=0.1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', C=1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', C=10, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', multi_class='multinomial', C=0.1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', multi_class='multinomial', C=1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, solver='lbfgs', multi_class='multinomial', C=10, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, penalty='l1', C=0.1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, penalty='l1', C=1, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)\n",
    "LRClassifier = LogisticRegression(random_state=42, penalty='l1', C=10, max_iter=5000)\n",
    "print_model_accuracy(LRClassifier, train_list, val_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 6))\n",
    "plt.bar(left=range(len(train_list)), height=train_list, width=0.25, color='skyblue')\n",
    "plt.bar(left=np.array(range(len(train_list))) + 0.4, height=val_list, width=0.25, color='wheat')\n",
    "plt.ylim(0, 0.5)\n",
    "plt.legend([\"train\", \"validation\"])\n",
    "plt.xticks(range(len(train_list)), [\"'lbfgs', C=0.1\", \"'lbfgs', C=1\", \"'lbfgs', C=10\", \n",
    "                                    \"'lbfgs', 'multiomial', C=0.1\",\n",
    "                                    \"'lbfgs', 'multiomial', C=1\",\n",
    "                                    \"'lbfgs', 'multiomial', C=10\",\n",
    "                                    \"penalty='l1', C=0.1\", \"penalty='l1', C=1\", \"penalty='l1', C=10\"], rotation='vertical')\n",
    "plt.xlabel(\"model with different hyperparameter\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.title(\"Performance of Logistic Regression\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"lg.jpg\", dpi=300)\n",
    "print(\"Best train accuracy: {}\".format(train_list[np.argmax(val_list)]))\n",
    "print(\"Best validation accuracy: {}\".format(max(val_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# search for different parameters\n",
    "train_list = []\n",
    "val_list = []\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=5)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=10)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=15)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=20)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=25)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)\n",
    "KNNClassifier = KNeighborsClassifier(n_neighbors=30)\n",
    "print_model_accuracy(KNNClassifier, train_list, val_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 6))\n",
    "plt.bar(left=range(len(train_list)), height=train_list, width=0.25, color='skyblue')\n",
    "plt.bar(left=np.array(range(len(train_list))) + 0.4, height=val_list, width=0.25, color='wheat')\n",
    "plt.ylim(0, 0.6)\n",
    "plt.legend([\"train\", \"validation\"])\n",
    "plt.xticks(range(len(train_list)), [\"5\", \"10\", \"15\", \"20\", \"35\", \"30\"])\n",
    "plt.xlabel(\"model with different hyperparameter $k$\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.title(\"Performance of K-nearest Neighbors\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"knn.jpg\", dpi=300)\n",
    "print(\"Best train accuracy: {}\".format(train_list[np.argmax(val_list)]))\n",
    "print(\"Best validation accuracy: {}\".format(max(val_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# search for different parameters\n",
    "train_list = []\n",
    "val_list = []\n",
    "\n",
    "RFClassifier = RandomForestClassifier(n_estimators=10, max_depth=3, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=10, max_depth=4, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=10, max_depth=5, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=10, max_depth=6, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "\n",
    "RFClassifier = RandomForestClassifier(n_estimators=50, max_depth=3, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=50, max_depth=4, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=50, max_depth=5, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=50, max_depth=6, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "\n",
    "RFClassifier = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=100, max_depth=4, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)\n",
    "RFClassifier = RandomForestClassifier(n_estimators=100, max_depth=6, random_state=42)\n",
    "print_model_accuracy(RFClassifier, train_list, val_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 6))\n",
    "plt.bar(left=range(len(train_list)), height=train_list, width=0.25, color='skyblue')\n",
    "plt.bar(left=np.array(range(len(train_list))) + 0.4, height=val_list, width=0.25, color='wheat')\n",
    "plt.ylim(0, 0.5)\n",
    "plt.legend([\"train\", \"validation\"])\n",
    "plt.xticks(range(len(train_list)), [\"# of tree=10, max_depth=3\", \"# of tree=10, max_depth=4\", \n",
    "                                    \"# of tree=10, max_depth=5\", \"# of tree=10, max_depth=6\",\n",
    "                                    \"# of tree=50, max_depth=3\", \"# of tree=50, max_depth=4\", \n",
    "                                    \"# of tree=50, max_depth=5\", \"# of tree=50, max_depth=6\",\n",
    "                                    \"# of tree=100, max_depth=3\", \"# of tree=100, max_depth=4\", \n",
    "                                    \"# of tree=100, max_depth=5\", \"# of tree=100, max_depth=6\",], rotation='vertical')\n",
    "plt.xlabel(\"model with different hyperparameter\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.title(\"Performance of Random Forest\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"rf.jpg\", dpi=300)\n",
    "print(\"Best train accuracy: {}\".format(train_list[np.argmax(val_list)]))\n",
    "print(\"Best validation accuracy: {}\".format(max(val_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "# search for different parameters\n",
    "train_list = []\n",
    "val_list = []\n",
    "\n",
    "SVMClassifier = SVC(C=1, kernel='linear', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=10, kernel='linear', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=100, kernel='linear', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "\n",
    "SVMClassifier = SVC(C=1, kernel='rbf', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=10, kernel='rbf', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=100, kernel='rbf', random_state=42)\n",
    "\n",
    "\n",
    "SVMClassifier = SVC(C=1, kernel='poly', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=10, kernel='poly', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=100, kernel='poly', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "\n",
    "SVMClassifier = SVC(C=1, kernel='sigmoid', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=10, kernel='sigmoid', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)\n",
    "SVMClassifier = SVC(C=100, kernel='sigmoid', random_state=42)\n",
    "print_model_accuracy(SVMClassifier, train_list, val_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 6))\n",
    "plt.bar(left=range(len(train_list)), height=train_list, width=0.25, color='skyblue')\n",
    "plt.bar(left=np.array(range(len(train_list))) + 0.4, height=val_list, width=0.25, color='wheat')\n",
    "plt.ylim(0, 0.5)\n",
    "plt.legend([\"train\", \"validation\"])\n",
    "plt.xticks(range(len(train_list)), [\"'linear', C=1\", \"'linear', C=10\", \"'linear', C=100\", \n",
    "                                    \"'rbf', C=1\", \"'rbf', C=10\", \"'rbf', C=100\",\n",
    "                                    \"'poly', C=1\", \"'poly', C=10\", \"'poly', C=100\",\n",
    "                                    \"'sigmoid', C=1\", \"'sigmoid', C=10\", \"'sigmoid', C=100\",], rotation='vertical')\n",
    "plt.xlabel(\"model with different hyperparameter\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.title(\"Performance of Support Vector Machine\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"svm.jpg\", dpi=300)\n",
    "print(\"Best train accuracy: {}\".format(train_list[np.argmax(val_list)]))\n",
    "print(\"Best validation accuracy: {}\".format(max(val_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply best model\n",
    "\n",
    "The best model we found is SVM model, with `rbf` kernel function and `C` = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train best model\n",
    "best_model = LogisticRegression(random_state=42, penalty='l1', C=10, max_iter=5000)\n",
    "best_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load test data\n",
    "test_df = pd.read_hdf(\"test_feature.h5\", \"feature\")\n",
    "X_test = feature_normalize(test_df)\n",
    "\n",
    "# predict and save results\n",
    "test_prediction = best_model.predict(X_test)\n",
    "pd.DataFrame(test_prediction).T.to_csv(\"submission.csv\", index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:rl]",
   "language": "python",
   "name": "conda-env-rl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}