16_Random_Forest_Clean_Prints_and_write_output_synt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:02.560705Z",
     "start_time": "2019-07-27T12:04:02.034265Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# IMPORTS\n",
    "# Pandas is used for data manipulation\n",
    "import pandas as pd\n",
    "# Use numpy to convert to arrays\n",
    "import numpy as np\n",
    "# Using Skicit-learn to split data into training and testing sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Import tools needed for visualization\n",
    "from sklearn.tree import export_graphviz\n",
    "# import pydot\n",
    "# Import the model we are using\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "# Utility to store and load model from disk\n",
    "from sklearn.externals import joblib\n",
    "# write csv files\n",
    "import csv\n",
    "# Import charting lib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:02.620102Z",
     "start_time": "2019-07-27T12:04:02.562768Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# UTIL\n",
    "def test_model(forest_model, test_features, test_labels):\n",
    "    # Use the forest's predict method on the test data\n",
    "    predictions = np.round(forest_model.predict(test_features))\n",
    "\n",
    "    # Calculate the absolute errors\n",
    "    errors = abs(predictions - test_labels)\n",
    "\n",
    "    # Print out the mean absolute error (mae)\n",
    "    print('Mean Absolute Error:', round(np.mean(errors), 2))\n",
    "\n",
    "    # Calculate mean absolute percentage error (MAPE)\n",
    "    mape = 100 * (errors / test_labels)\n",
    "\n",
    "    # Calculate and display accuracy\n",
    "    accuracy = 100 - np.mean(mape)\n",
    "    print('Accuracy:', round(accuracy, 2), '%.')\n",
    "\n",
    "    # Pull out one tree from the forest\n",
    "    #tree = forest_model.estimators_[5]\n",
    "    # print('The depth of this tree is:', tree.tree_.max_depth)\n",
    "\n",
    "    total_trues = sum(x == 2 for x in test_labels)\n",
    "    total_predictions = sum(x == 2 for x in predictions)\n",
    "    total_errors = sum(x == 1 for x in errors)\n",
    "    print('Total Samples:', len(test_labels))\n",
    "    print('Total Trues:', total_trues)\n",
    "    print('Total Predictions:', total_predictions)\n",
    "    print('Total Errors:', total_errors)\n",
    "\n",
    "    false_positive = sum(predict > label for predict, label in zip(predictions, test_labels))\n",
    "    false_negative = sum(predict < label for predict, label in zip(predictions, test_labels))\n",
    "    true_positive = total_predictions - false_positive\n",
    "    print('false_positive:', false_positive)\n",
    "    print('false_negative:', false_negative)\n",
    "    print('true_positive:', true_positive)\n",
    "    \n",
    "    print('>>>>>>>>>>>>>>>>>>>>')\n",
    "    for index, value in enumerate(zip(predictions, test_labels)):\n",
    "        if value[0] < value[1]:\n",
    "            print(test_features_[index,0])\n",
    "    print('>>>>>>>>>>>>>>>>>>>>')\n",
    "    \n",
    "    \n",
    "    precision = true_positive / total_predictions\n",
    "    recall = true_positive / (true_positive + false_negative)\n",
    "    print('precision:', precision)\n",
    "    print('recall:', recall)\n",
    "    return precision, recall\n",
    "\n",
    "def draw_tree(forest_model, test_features, test_labels): \n",
    "    importances = forest_model.feature_importances_\n",
    "    std = np.std([tree.feature_importances_ for tree in forest_model.estimators_],\n",
    "                 axis=0)\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "\n",
    "    # Print the feature ranking\n",
    "    print(\"Feature ranking:\")\n",
    "\n",
    "    for f in range(test_features.shape[1]):\n",
    "        print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "    # Plot the feature importances of the forest\n",
    "    plt.figure()\n",
    "    plt.title(\"Feature importances\")\n",
    "    plt.bar(range(test_features.shape[1]), importances[indices],\n",
    "           color=\"r\", yerr=std[indices], align=\"center\")\n",
    "    plt.xticks(range(test_features.shape[1]), indices)\n",
    "    plt.xlim([-1, test_features.shape[1]])\n",
    "    plt.show()    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:02.826173Z",
     "start_time": "2019-07-27T12:04:02.621537Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fragmentAndSide',\n",
       " 'fragmentTotal',\n",
       " 'fragmentVote',\n",
       " 'devideVoteByTotal',\n",
       " 'fragmentAndSideVote',\n",
       " 'devideSideVoteBySideTotal',\n",
       " 'fragmentAndSideTrendVote',\n",
       " 'devideSideTrendVoteBySideTotal',\n",
       " 'fragmentAndSideTrendVoteStrict',\n",
       " 'devideSideTrendVoteStrictBySideTotal',\n",
       " 'fragmentAndSideTrendVoteSync',\n",
       " 'devideSideTrendVoteSyncBySideTotal',\n",
       " 'votesOverlapMax',\n",
       " 'divideOverlapMaxBySideTotal',\n",
       " 'votesOverlapHeight',\n",
       " 'votesSupportOverlapRect',\n",
       " 'divideSupportOverlapBySideTotal',\n",
       " 'divideSupportOverlapBySideVote']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare the source data\n",
    "features = pd.read_csv('20181216_034921_pairs_final_enhanced_synt.csv') #20181020_212330_pairs_votes_enhanced\n",
    "\n",
    "# Remove the irrelevant texts from the features\n",
    "# axis 1 refers to the columns\n",
    "# features = features.drop('fragmanetAndSide', axis = 1)\n",
    "features = features.drop('fragment', axis = 1)\n",
    "features = features.drop('fragmentAndSideTotal', axis = 1)\n",
    "features = features.drop('fragmentAndSideTrend', axis = 1)\n",
    "features = features.drop('fragmentAndSideCubes', axis = 1)\n",
    "features = features.drop('fragmentAndSideDrawRect', axis = 1)\n",
    "features = features.drop('fragmentAndSideMatchPoint', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideCubes', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideDrawRect', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideMatchPoint', axis = 1)\n",
    "features = features.drop('origCoordinates', axis = 1)\n",
    "features = features.drop(\"firstFileName\", axis = 1)\n",
    "features = features.drop(\"firstCroppedWidth\", axis = 1)\n",
    "features = features.drop(\"firstOffsetX\", axis = 1)\n",
    "features = features.drop(\"firstOffsetY\", axis = 1)\n",
    "features = features.drop(\"firstHorizontalFlip\", axis = 1)\n",
    "features = features.drop(\"secondFileName\", axis = 1)\n",
    "features = features.drop(\"secondCroppedWidth\", axis = 1)\n",
    "features = features.drop(\"secondOffsetX\", axis = 1)\n",
    "features = features.drop(\"secondOffsetY\", axis = 1)\n",
    "features = features.drop(\"secondHorizontalFlip\", axis = 1)\n",
    "features = features.drop(\"prediction\", axis = 1)\n",
    "\n",
    "# features = features.drop('fragmentTotal', axis = 1)\n",
    "# features = features.drop('fragmentAndSideVote', axis = 1)\n",
    "# features = features.drop('fragmentAndSideTrendVote', axis = 1)\n",
    "\n",
    "# One-hot encode categorical features\n",
    "#features = pd.get_dummies(features)\n",
    "\n",
    "# Labels are the values we want to predict\n",
    "labels = np.array(features['class'])\n",
    "labels = labels + 1\n",
    "\n",
    "# Remove the labels from the features\n",
    "# axis 1 refers to the columns\n",
    "features= features.drop('class', axis = 1)\n",
    "\n",
    "# Saving feature names for later use\n",
    "feature_list = list(features.columns)\n",
    "\n",
    "# Convert to numpy array\n",
    "features = np.array(features)\n",
    "\n",
    "feature_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:29.305565Z",
     "start_time": "2019-07-27T12:04:27.103329Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 99.94 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 38\n",
      "Total Errors: 1\n",
      "false_positive: 0\n",
      "false_negative: 1\n",
      "true_positive: 38\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "PX303Fg006_7X3_5X0_1_PX303Fg006_7X3_6X0_0\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 0.9743589743589743\n",
      "Feature ranking:\n",
      "1. feature 11 (0.297117)\n",
      "2. feature 12 (0.273536)\n",
      "3. feature 14 (0.202120)\n",
      "4. feature 15 (0.133027)\n",
      "5. feature 5 (0.019826)\n",
      "6. feature 3 (0.019373)\n",
      "7. feature 16 (0.017322)\n",
      "8. feature 6 (0.010095)\n",
      "9. feature 4 (0.007893)\n",
      "10. feature 13 (0.004816)\n",
      "11. feature 0 (0.004038)\n",
      "12. feature 1 (0.002786)\n",
      "13. feature 9 (0.002460)\n",
      "14. feature 7 (0.002417)\n",
      "15. feature 2 (0.001291)\n",
      "16. feature 8 (0.000954)\n",
      "17. feature 10 (0.000930)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFaFJREFUeJzt3XuUpHV95/H3xxlAbst1QGAGBhU4sqxRMiK7Gu2IJoAKxsQNRBPddWWTs6yyGg3qLoeQeI6ocS8nxATE6FEREQ1OdFwwUdxdE5ABAbmIDhczw3VEMAQv3L77x/OMKdqe6eqqmunp37xf59SZqufyrW89Xf3p3/N7qntSVUiS2vKU+W5AkjR5hrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd20Tkvx5kv82331IW0r8nLs2JckdwL7A4wOLD62qu8aoOQV8oqqWjtfdwpTko8C6qvqv892L2uXIXcN4ZVXtMnAbOdgnIcni+Xz+cSRZNN89aNtguGtkSY5O8ndJHkxyXT8i37Du3yW5OclDSW5L8h/75TsDXwL2T/JP/W3/JB9N8scD+08lWTfw+I4kf5DkeuDhJIv7/T6bZH2S25O8eRO9/qz+htpJ3pHkviR3J3lVkuOTfCfJD5K8a2DfM5NcnOTT/eu5JskvDKx/VpLL++NwY5ITpj3vh5KsSvIw8EbgtcA7+tf+1/12pye5ta9/U5JfG6jxhiT/L8kHkjzQv9bjBtbvmeQvk9zVr79kYN0rklzb9/Z3SZ49sO4PktzZP+ctSY4Z4suuhaKqvHnb6A24A3jpDMsPAO4HjqcbJLysf7ykX/9y4BlAgBcDPwKO7NdN0U1LDNb7KPDHA4+ftE3fx7XAMmDH/jmvBs4AtgeeDtwG/OpGXsfP6ve1H+v33Q54E7AeuADYFfiXwI+Bg/vtzwQeBX6j3/73gdv7+9sBa4B39X28BHgIOGzgeX8IvKDv+anTX2u/3WuA/fttfhN4GNivX/eG/vnfBCwCfg+4i3+eVv0i8Glgj76fF/fLnwvcBzy/3+/1/XHcATgMWAvs32+7HHjGfL/fvE3u5shdw7ikH/k9ODAqfB2wqqpWVdUTVfVlYDVd2FNVX6yqW6vzNeAy4JfG7ON/VdXaqvox8Dy6HyRnVdUjVXUbcB5w0pC1HgXeU1WPAhcCewP/s6oeqqobgZuAXxjY/uqqurjf/oN0IX10f9sFeG/fx1eALwAnD+z7+ar6en+cfjJTM1X1maq6q9/m08B3gaMGNvleVZ1XVY8DHwP2A/ZNsh9wHPC7VfVAVT3aH2+AU4C/qKorq+rxqvoY8NO+58fpQv7wJNtV1R1VdeuQx04LgOGuYbyqqnbvb6/qlx0EvGYg9B8EXkgXOiQ5LskV/RTHg3Shv/eYfawduH8Q3dTO4PO/i+7i7zDu74MSulE6wL0D639MF9o/99xV9QSwjm6kvT+wtl+2wffozmxm6ntGSX5nYPrkQeAInny87hl4/h/1d3ehO5P5QVU9MEPZg4C3TTtGy+hG62uA0+jOSu5LcmGS/WfrUwuH4a5RrQU+PhD6u1fVzlX13iQ7AJ8FPgDsW1W7A6vopmgAZvqI1sPATgOPnzbDNoP7rQVun/b8u1bV8WO/spkt23AnyVOApXRTI3cBy/plGxwI3LmRvn/ucZKD6M46TgX26o/XDfzz8dqUtcCeSXbfyLr3TDtGO1XVpwCq6oKqeiHdD4ECzh7i+bRAGO4a1SeAVyb51SSLkjy1v1C5lG7ueQe6eezH+ot/vzKw773AXkl2G1h2LXB8f3HwaXSjyk35BvBQf1Fwx76HI5I8b2Kv8Ml+Mcmr031S5zS66Y0rgCvprie8I8l2/UXlV9JN9WzMvXTXCDbYmS5c10N3MZpu5D6rqrqb7gL1nyXZo+/hRf3q84DfTfL8dHZO8vIkuyY5LMlL+h/EP6E7U3liI0+jBchw10iqai1wIt1UyHq6UeLbgadU1UPAm4GLgAeA3wJWDuz7beBTwG39dMH+wMeB6+gu+F1Gd4FwU8//OPAK4Dl0Fze/D3wY2G1T+43h83QXOh8Afht4dT+//QhdmB/X9/BnwO/0r3Fjzqeb634wySVVdRPwJ8Df0wX/vwK+PofefpvuGsK36S6gngZQVavpLsL+ad/3GrqLs9D98H1v3/M9wD7AO+fwnNrK+UtM0iySnAk8s6peN9+9SMNy5C5JDTLcJalBTstIUoMcuUtSg+btDzDtvffetXz58vl6eklakK6++urvV9WS2babt3Bfvnw5q1evnq+nl6QFKcn3htnOaRlJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgxZ8uE9NTTE1NTXfbUjSVmXBh7sk6ecZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhnvPz8tLaonhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDRXuSY5NckuSNUlO38g2/zbJTUluTHLBZNuUJM3F4tk2SLIIOAd4GbAOuCrJyqq6aWCbQ4B3Ai+oqgeS7LO5GpYkzW6YkftRwJqquq2qHgEuBE6cts2bgHOq6gGAqrpvsm1KkuZimHA/AFg78Hhdv2zQocChSb6e5Iokx85UKMkpSVYnWb1+/frROpYkzWpSF1QXA4cAU8DJwHlJdp++UVWdW1UrqmrFkiVLJvTUkqTphgn3O4FlA4+X9ssGrQNWVtWjVXU78B26sJckzYNhwv0q4JAkByfZHjgJWDltm0voRu0k2Ztumua2CfYpSZqDWcO9qh4DTgUuBW4GLqqqG5OcleSEfrNLgfuT3AR8FXh7Vd2/uZqWJG3arB+FBKiqVcCqacvOGLhfwFv7myRpnvkbqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6bwdTUFFNTU/PdhqRtmOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgocI9ybFJbkmyJsnpm9ju15NUkhWTa1GSNFezhnuSRcA5wHHA4cDJSQ6fYbtdgbcAV066SUnS3Awzcj8KWFNVt1XVI8CFwIkzbPdHwNnATybYnyRpBMOE+wHA2oHH6/plP5PkSGBZVX1xU4WSnJJkdZLV69evn3Oz25qpqSmmpqbmuw1JC9DYF1STPAX4IPC22batqnOrakVVrViyZMm4Ty1J2ohhwv1OYNnA46X9sg12BY4ALk9yB3A0sNKLqpI0f4YJ96uAQ5IcnGR74CRg5YaVVfXDqtq7qpZX1XLgCuCEqlq9WTqWJM1q1nCvqseAU4FLgZuBi6rqxiRnJTlhczcoSZq7xcNsVFWrgFXTlp2xkW2nxm9LkjQOf0NVkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJatBQ/4fqvEtG36Zqsr1I0gLgyF2SGmS4S1KDDHdJapDhLkkNWhgXVCdptouzm1rvxVlJC4Qjd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYNFe5Jjk1yS5I1SU6fYf1bk9yU5Pokf5vkoMm3Kkka1qzhnmQRcA5wHHA4cHKSw6dt9k1gRVU9G7gYeN+kG5UkDW+YkftRwJqquq2qHgEuBE4c3KCqvlpVP+ofXgEsnWybkqS5GOY/6zgAWDvweB3w/E1s/0bgSzOtSHIKcArAgQceOGSLW6nZ/tOPTW3jf/ohaTOb6AXVJK8DVgDvn2l9VZ1bVSuqasWSJUsm+dSSpAHDjNzvBJYNPF7aL3uSJC8F3g28uKp+Opn2JEmjGGbkfhVwSJKDk2wPnASsHNwgyXOBvwBOqKr7Jt+mJGkuZg33qnoMOBW4FLgZuKiqbkxyVpIT+s3eD+wCfCbJtUlWbqScJGkLGGZahqpaBayatuyMgfsvnXBfkqQx+BuqktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho01J8f0GY229+G39R6/za8pBk4cpekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNd82Zqaoqpqan5bkNqkuEuSQ0y3LcRkxwlO+KWtn6GuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4a4m+PFM6ckMd0lqkOEuDfAMQK0w3CWpQYa7JDXIcJekBhnu0mbi/L3m0+L5bkATloy+vmr4OpvaZrDOJHvahm34IXH55ZfPax9aOIYauSc5NsktSdYkOX2G9Tsk+XS//sokyyfdqKTxbY1/+tkznM1j1pF7kkXAOcDLgHXAVUlWVtVNA5u9EXigqp6Z5CTgbOA3N0fD2oZN8mxCatww0zJHAWuq6jaAJBcCJwKD4X4icGZ//2LgT5Okyu8obaUmOVW0NU6FbaMmOX01qVrzNaU2TLgfAKwdeLwOeP7Gtqmqx5L8ENgL+P7gRklOAU4BOPDAA4fvclNv3A2nc8MeuI3VmlSd+expkrW2xte3NfY0yVqTfH3a5m3RC6pVdS5wLsCKFSscakjaak1qpD1fF8GHCfc7gWUDj5f2y2baZl2SxcBuwP0T6VDSxLT+aZvWX99cDBPuVwGHJDmYLsRPAn5r2jYrgdcDfw/8BvAV59ulydkaQ2uhj2xbN2u493PopwKXAouAj1TVjUnOAlZX1UrgfODjSdYAP6D7ASBt0wwtzaeh5tyrahWwatqyMwbu/wR4zWRbkySNyt9QlQY42lYrDHc1wVCWnsw/HCZJDTLcJalBTstsIyY5beEUiLT1c+QuSQ0y3CWpQU7LaN44vSNtPo7cJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgBf+Hw/zjU5L08xy5S1KDDHdJatCCn5aZFKd3JLXEkbskNchwl6QGOS2zFXOqSNKoDPfNwFCWNN+clpGkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAalqubniZP1wPcmVG5v4PtbUZ1J1toae5pkLXva8rXsacvXmmRPB1XVktk2mrdwn6Qkq6tqxdZSp/WeJlnLnrZ8LXva8rUm2dOwnJaRpAYZ7pLUoFbC/dytrM4ka22NPU2ylj1t+Vr2tOVrTbKnoTQx5y5JerJWRu6SpAGGuyQ1aEGFe5KPJLkvyQ0Dy16T5MYkTyQZ+qNGG6n1/iTfTnJ9kr9KsvuotQbWvS1JJdl7xJ7OTHJnkmv72/HDvsZpte9I8q2+xuoRazw1yTeSXNcf8z+c4/4zHqck/7k/7jcmed8Ife2e5OK+xs1J/vVca0yrtyjJN5N8YY77zfT1+6P+/XRtksuS7D9CP8cmuSXJmiSnz3X/TfU3Rq23JLmh/5qdNkadwwbe29cm+cdR6yVZluSrSW7q+3rLGH39l77GDUk+leSpc9h3pvfBnkm+nOS7/b97jNrb0KpqwdyAFwFHAjcMLHsWcBhwObBizFq/Aizu758NnD1qrX75MuBSul/W2nvEns4Efn8Cx+6OYXqYpUaAXfr72wFXAkePecx/GfgbYIf+8T4j9PUx4D/097cHdh/zdb4VuAD4wgTen/9i4P6bgT+fY81FwK3A0/vXdh1w+Iiva8b36Qh1jgBuAHai+686/wZ45gTeo4uAe+h+SWeU/fcDjuzv7wp8Z5RjBRwA3A7s2D++CHjDmO+D9wGn9/dPHzZbxrktqJF7Vf0f4AfTlt1cVbdMqNZlVfVY//AKYOmotXr/HXgHMNRV603U2SpU55/6h9v1t6GvyG/k9f0e8N6q+mm/zX1z6SnJbnTfTOf3+z9SVQ/Opca0ekuBlwMfnuu+G3lP/ePAw52Zw/HqHQWsqarbquoR4ELgxLn2trH+RvQs4Mqq+lH//fI14NUTqHsMcGtVjfSb61V1d1Vd099/CLiZLqhHsRjYMcliuh9id82hj5mO84l0gxD6f181Yl9DW1DhvoX9e+BLo+6c5ETgzqq6bgK9nNqf2n9kjNO5Ai5LcnWSU0ZtpJ+yuBa4D/hyVV05aq3eocAvJbkyydeSPG+O+x8MrAf+sp9K+XCSncfo53/Q/UB+YowaT5LkPUnWAq8Fzpjj7gcAawcer2P0wJqUG+i+Znsl2Qk4nu4sdVwnAZ+aQB2SLAeeS3d2OSdVdSfwAeAfgLuBH1bVZWO2tG9V3d3fvwfYd8x6szLcZ5Dk3cBjwCdH3H8n4F3M/Rt5Jh8CngE8h+6N9icj1nlhVR0JHAf8pyQvGqVIVT1eVc+hO6s5KskRI/azwWJgT+Bo4O3ARUkyx/2PBD5UVc8FHqY77Z2zJK8A7quqq0fZf2Oq6t1VtYzu/XTqJGvPh6q6mW7a8jLgfwPXAo+PUzPJ9sAJwGfG7S/JLsBngdOmnTkNu/8edCPtg4H9gZ2TvG7cvjaobm5ms38G3XCfJskbgFcAr+2/CKN4Bt0b47okd9AF4TVJnjbXQlV1bx+oTwDn0Z2mz1k/Gtkw7fFXo9YZqPcg8FXg2HHq0I1EP9dP+XyDbsQ868XnafuvGziDuJgu7EfxAuCE/mt2IfCSJJ8YsdZMPgn8+hz3uZMnj4qX9svmVVWdX1W/WFUvAh6gm98ex3HANVV17zhFkmxHF+yfrKrPjVjmpcDtVbW+qh4FPgf8m3H6Au5Nsl/f4350Z76bleE+IMmxdKfkJ1TVj0atU1Xfqqp9qmp5VS2nC6Ajq+qeEXrab+Dhr9GdEs+1xs5Jdt1wn+7C8Sh1lqT/BFGSHYGXAd+ea51pLqG7qEqSQ+kuGg791/P6Y7o2yWH9omOAm0ZppKreWVVL+6/ZScBXqmqsEVuSQwYensjcj9dVwCFJDu5HtycBK8fpaRKS7NP/eyDdfPsFY5Y8mTGnZPozvvOBm6vqg2OU+gfg6CQ79TWPoZu/H8dK4PX9/dcDnx+z3uw29xXbSd7ovvh3A4/SBeYb6QJvHfBT4F7g0jFqraGb37y2vw31yYaZak1bfwfDfVpmpp4+DnwLuJ7uDbLfCMft6XSfsrgOuBF494jH/9nAN/tebgDOmMDXb3vgE329a4CXjNDXc4DVfV+XAHtM4L02xdw/LTPT6/ts/9quB/4aOGCEXo6nGxnfOurXbpj36Rxr/V+6H6LXAceMeax3Bu4HdhuzzgvppjuuH/gePn7EWn9I94P4hv57cIcx3wd7AX8LfJfu00V7jvsene3mnx+QpAY5LSNJDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoP+P5lRelnYoPg0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Random Forest 1st model\n",
    "# Split the data into training and testing sets\n",
    "train_features_, test_features_, train_labels, test_labels = train_test_split(features, labels, test_size = 0.3)\n",
    "\n",
    "train_features = train_features_[:,1:] # remove the fragmentAndSide column which is the label\n",
    "test_features = test_features_[:,1:] # remove the fragmentAndSide column which is the label\n",
    "\n",
    "# Instantiate model \n",
    "rf = RandomForestClassifier(n_estimators= 1000, random_state=42, n_jobs=-1)\n",
    "\n",
    "rf.fit(train_features, train_labels)\n",
    "\n",
    "test_model(rf, test_features, test_labels)\n",
    "draw_tree(rf, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:38.229028Z",
     "start_time": "2019-07-27T12:04:35.904280Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 99.94 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 38\n",
      "Total Errors: 1\n",
      "false_positive: 0\n",
      "false_negative: 1\n",
      "true_positive: 38\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "PX303Fg006_7X3_5X0_1_PX303Fg006_7X3_6X0_0\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 0.9743589743589743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.9743589743589743)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AdaBoostClassifier model \n",
    "ab = AdaBoostClassifier(n_estimators= 1000, learning_rate=0.8)\n",
    "\n",
    "ab.fit(train_features, train_labels)\n",
    "\n",
    "test_model(ab, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:41.459849Z",
     "start_time": "2019-07-27T12:04:38.231167Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 99.94 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 38\n",
      "Total Errors: 1\n",
      "false_positive: 0\n",
      "false_negative: 1\n",
      "true_positive: 38\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "PX303Fg006_7X3_5X0_1_PX303Fg006_7X3_6X0_0\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 0.9743589743589743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.9743589743589743)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# BaggingClassifier model \n",
    "bc = BaggingClassifier(n_estimators= 1000)\n",
    "\n",
    "bc.fit(train_features, train_labels)\n",
    "\n",
    "test_model(bc, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:42.308803Z",
     "start_time": "2019-07-27T12:04:41.461666Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 100.0 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 39\n",
      "Total Errors: 0\n",
      "false_positive: 0\n",
      "false_negative: 0\n",
      "true_positive: 39\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 1.0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ExtraTreesClassifier model \n",
    "et = ExtraTreesClassifier(n_estimators= 1000)\n",
    "\n",
    "et.fit(train_features, train_labels)\n",
    "\n",
    "test_model(et, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:42.637738Z",
     "start_time": "2019-07-27T12:04:42.311082Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 99.94 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 38\n",
      "Total Errors: 1\n",
      "false_positive: 0\n",
      "false_negative: 1\n",
      "true_positive: 38\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "PX303Fg006_7X3_5X0_1_PX303Fg006_7X3_6X0_0\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 0.9743589743589743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.9743589743589743)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GradientBoostingClassifier model \n",
    "gb = GradientBoostingClassifier(n_estimators= 1000)\n",
    "\n",
    "gb.fit(train_features, train_labels)\n",
    "\n",
    "test_model(gb, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-27T12:04:47.665147Z",
     "start_time": "2019-07-27T12:04:42.639459Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 0.0\n",
      "Accuracy: 99.94 %.\n",
      "Total Samples: 858\n",
      "Total Trues: 39\n",
      "Total Predictions: 38\n",
      "Total Errors: 1\n",
      "false_positive: 0\n",
      "false_negative: 1\n",
      "true_positive: 38\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "PX303Fg006_7X3_5X0_1_PX303Fg006_7X3_6X0_0\n",
      ">>>>>>>>>>>>>>>>>>>>\n",
      "precision: 1.0\n",
      "recall: 0.9743589743589743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.9743589743589743)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# VotingClassifier model \n",
    "vt = VotingClassifier(estimators=[('ab', ab), ('rf', rf), ('bc', bc), ('et', et), ('gb', gb)], voting='soft', n_jobs=-1)\n",
    "\n",
    "vt.fit(train_features, train_labels)\n",
    "\n",
    "test_model(vt, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import cross_val_predict\n",
    "# from sklearn import metrics\n",
    "# predicted = cross_val_predict(rf, test_features, test_labels, cv=5)\n",
    "# metrics.accuracy_score(test_labels, np.round(predicted)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Random Forest 2nd variation of model - just for reference - not used\n",
    "rf_new = RandomForestRegressor(n_estimators = 1000, criterion = 'mse', max_depth = None, \n",
    "                               min_samples_split = 2, min_samples_leaf = 1)\n",
    "rf_new.fit(train_features, train_labels)\n",
    "\n",
    "test_model(rf_new, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Random Forest 3rd model - Limit depth of tree to 2 levels - not used\n",
    "rf_small = RandomForestRegressor(n_estimators=10, max_depth = 3, random_state=42)\n",
    "rf_small.fit(train_features, train_labels)\n",
    "\n",
    "test_model(rf_small, test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rndFstBasic2.pkl']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finally - use the 1st model and this time train on the entire set \n",
    "fit_features = features[:,1:] # remove the fragmentAndSide column which is the label\n",
    "\n",
    "rf.fit(fit_features, labels);\n",
    "\n",
    "joblib.dump(rf, 'rndFstBasic3.pkl') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Run on the output of the voting and classify them\n",
    "# Read in data as pandas dataframe\n",
    "orig_features = pd.read_csv('20181216_034921_pairs_final_enhanced_synt.csv') #('20181213_031300_pairs_final_flipped_enhanced.csv')\n",
    "\n",
    "# Remove the irrelevant texts from the features\n",
    "# axis 1 refers to the columns\n",
    "features = orig_features.drop('fragmentAndSide', axis = 1)\n",
    "features = features.drop('fragment', axis = 1)\n",
    "features = features.drop('fragmentAndSideTotal', axis = 1)\n",
    "features = features.drop('fragmentAndSideTrend', axis = 1)\n",
    "features = features.drop('fragmentAndSideCubes', axis = 1)\n",
    "features = features.drop('fragmentAndSideDrawRect', axis = 1)\n",
    "features = features.drop('fragmentAndSideMatchPoint', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideCubes', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideDrawRect', axis = 1)\n",
    "features = features.drop('rotateFragmentAndSideMatchPoint', axis = 1)\n",
    "features = features.drop('origCoordinates', axis = 1)\n",
    "features = features.drop(\"firstFileName\", axis = 1)\n",
    "features = features.drop(\"firstCroppedWidth\", axis = 1)\n",
    "features = features.drop(\"firstOffsetX\", axis = 1)\n",
    "features = features.drop(\"firstOffsetY\", axis = 1)\n",
    "features = features.drop(\"firstHorizontalFlip\", axis = 1)\n",
    "features = features.drop(\"secondFileName\", axis = 1)\n",
    "features = features.drop(\"secondCroppedWidth\", axis = 1)\n",
    "features = features.drop(\"secondOffsetX\", axis = 1)\n",
    "features = features.drop(\"secondOffsetY\", axis = 1)\n",
    "features = features.drop(\"secondHorizontalFlip\", axis = 1)\n",
    "features = features.drop(\"class\", axis = 1)\n",
    "features = features.drop(\"prediction\", axis = 1)\n",
    "\n",
    "forest_model = joblib.load('rndFstBasic2.pkl') \n",
    "\n",
    "predictions = np.round(forest_model.predict(features))-1\n",
    "orig_features[\"prediction\"] = predictions\n",
    "filtered = orig_features # [orig_features[\"prediction\"] == 1]\n",
    "filtered.to_csv('20181216_034921_pairs_final_enhanced_final_synt2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_code_all_hidden": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}