Visualize_LOO_Experiments-Combine_Inf_Loss_Random.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.179372,
     "end_time": "2018-11-19T22:28:55.938944",
     "exception": false,
     "start_time": "2018-11-19T22:28:55.759572",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.448422,
     "end_time": "2018-11-19T22:28:56.387465",
     "exception": false,
     "start_time": "2018-11-19T22:28:55.939043",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import sklearn.metrics\n",
    "import scipy\n",
    "import adjustText\n",
    "import matplotlib.ticker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.011373,
     "end_time": "2018-11-19T22:28:56.398937",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.387564",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the default plot style\n",
    "#default_plt_width = 15\n",
    "#default_plt_height = 10\n",
    "#plt.rcParams['figure.figsize'] = [default_plt_width, default_plt_height]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.07511,
     "end_time": "2018-11-19T22:28:56.474093",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.398983",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"paper\")\n",
    "sns.set(font_scale=1.1)\n",
    "sns.despine(left=True)\n",
    "sns.set_style(\"ticks\", {\"xtick.major.size\": 8, \"ytick.major.size\": 8})\n",
    "cmap = sns.color_palette(\"Set1\")\n",
    "sns.palplot(cmap)\n",
    "sns.set_palette(cmap)\n",
    "plt_y_axis_fmt_string = '%.3f'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01699,
     "end_time": "2018-11-19T22:28:56.491135",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.474145",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "filename_prefix = \"aug_results_MNIST_3_vs_8_translate_10\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.013745,
     "end_time": "2018-11-19T22:28:56.504983",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.491238",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "filename_prefix = \"aug_results_NORB_0_vs_1_crop_10\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.011987,
     "end_time": "2018-11-19T22:28:56.517017",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.505030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data = np.load(\"{}.npz\".format(filename_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01365,
     "end_time": "2018-11-19T22:28:56.530715",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.517065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_acc = runs_data[\"no_aug_no_poison_acc\"]\n",
    "poisoned_acc = runs_data[\"poisoned_acc\"]\n",
    "all_aug_train_poisoned_acc = runs_data[\"all_aug_train_poisoned_acc\"]\n",
    "n_aug_sample_points = runs_data[\"n_aug_sample_points\"]\n",
    "n_train = runs_data[\"n_train\"]\n",
    "VSV_acc = runs_data[\"VSV_acc\"]\n",
    "is_SV = runs_data[\"is_SV\"].astype(np.int)\n",
    "n_SV = np.sum(is_SV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.012807,
     "end_time": "2018-11-19T22:28:56.543571",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.530764",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_inf = pd.read_pickle(\"{}.pkl\".format(filename_prefix))\n",
    "runs_data_loss = pd.read_pickle(\"{}_loss.pkl\".format(filename_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.012016,
     "end_time": "2018-11-19T22:28:56.555638",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.543622",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_inf[\"score\"] = \"influence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.011859,
     "end_time": "2018-11-19T22:28:56.567545",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.555686",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_loss[\"score\"] = \"loss\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.012942,
     "end_time": "2018-11-19T22:28:56.580541",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.567599",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df_unprocessed = pd.concat([\n",
    "    runs_data_inf,\n",
    "    runs_data_loss,\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.02403,
     "end_time": "2018-11-19T22:28:56.604620",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.580590",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df_unprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.021002,
     "end_time": "2018-11-19T22:28:56.625724",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.604722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline = run_df_unprocessed.query(\"test_type == 'baseline'\").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.012203,
     "end_time": "2018-11-19T22:28:56.637976",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.625773",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline[\"score\"] = \"baseline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.012462,
     "end_time": "2018-11-19T22:28:56.650492",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.638030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline[\"test_type\"] = \"Baseline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.023893,
     "end_time": "2018-11-19T22:28:56.674433",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.650540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.039035,
     "end_time": "2018-11-19T22:28:56.713517",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.674482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prop_inf = run_df_unprocessed.query(\"test_type == 'random_proportional' & score == 'influence'\")\n",
    "prop_inf[\"test_type\"] = \"Random Proportional Influence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.033651,
     "end_time": "2018-11-19T22:28:56.747263",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.713612",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prop_loss = run_df_unprocessed.query(\"test_type == 'random_proportional' & score == 'loss'\")\n",
    "prop_loss[\"test_type\"] = \"Random Proportional Loss\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01299,
     "end_time": "2018-11-19T22:28:56.760303",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.747313",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df = pd.concat([\n",
    "    baseline,\n",
    "    prop_inf,\n",
    "    prop_loss,\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.010931,
     "end_time": "2018-11-19T22:28:56.771279",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.760348",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df = run_df.rename(\n",
    "    index=str,\n",
    "    columns={\"test_accuracy\": \"Test Accuracy\",\n",
    "             \"n_auged\": \"Number of Augmented Points\",\n",
    "            },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.027225,
     "end_time": "2018-11-19T22:28:56.798744",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.771519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.013906,
     "end_time": "2018-11-19T22:28:56.812698",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.798792",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "VSV_x = n_SV\n",
    "VSV_y = VSV_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.586333,
     "end_time": "2018-11-19T22:28:57.399077",
     "exception": false,
     "start_time": "2018-11-19T22:28:56.812744",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "run_plot = sns.lineplot(x=\"Number of Augmented Points\",\n",
    "                        y=\"Test Accuracy\",\n",
    "                        hue=\"test_type\",\n",
    "                        style=\"test_type\",\n",
    "                        ci=95,\n",
    "                        data=run_df,\n",
    "                        markers=True,\n",
    "                        dashes=True,\n",
    "                        ax=ax)\n",
    "run_plot.scatter(VSV_x, VSV_y, marker=\"x\", color=\"k\", s=20)\n",
    "# text = run_plot.annotate(\"VSV\", (VSV_x, VSV_y))\n",
    "text = run_plot.text(VSV_x, VSV_y, \"VSV\", fontsize=12)\n",
    "l = ax.legend()\n",
    "#l.texts[0].set_text(\"\")\n",
    "#l.set_title('Whatever you want')\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "ax.legend(handles=handles[1:], labels=labels[1:])\n",
    "ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='11.5') # for legend text \n",
    "#run_plot.axhline(y=baseline_acc,\n",
    "#                 color=\"b\",\n",
    "#                 linestyle=\"--\",\n",
    "#                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "adjustText.adjust_text([text],\n",
    "                       x=[VSV_x],\n",
    "                       y=[VSV_y],\n",
    "                       add_objects=[run_plot],\n",
    "                       expand_points=(0.2, 0.2),\n",
    "                       expand_objects=(0.3, 0.3),\n",
    "                       ax=ax,\n",
    "                       force_objects=(0.1, 0.1))\n",
    "run_plot.get_figure().savefig(filename_prefix + \"_joined.pdf\",\n",
    "                              bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 2e-06,
     "end_time": "2018-11-19T22:28:57.399177",
     "exception": null,
     "start_time": "2018-11-19T22:28:57.399175",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 3e-06,
     "end_time": "2018-11-19T22:28:57.410801",
     "exception": null,
     "start_time": "2018-11-19T22:28:57.410798",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 3e-06,
     "end_time": "2018-11-19T22:28:57.421939",
     "exception": null,
     "start_time": "2018-11-19T22:28:57.421936",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "papermill": {
   "duration": 3.311604,
   "end_time": "2018-11-19T22:28:58.539623",
   "environment_variables": {},
   "exception": false,
   "output_path": "Visualize_LOO_Experiments-Combine_Inf_Loss_Random.ipynb",
   "parameters": {
    "filename_prefix": "aug_results_NORB_0_vs_1_crop_10"
   },
   "start_time": "2018-11-19T22:28:55.228019",
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}