create aggregate thanklove df.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Goal \n",
    "### glue together many df.\n",
    "\n",
    "### Stats\n",
    "+ What are summary statistics of each feature?\n",
    "+ How has thanking changed over time?\n",
    "+ How is gratitude distributes among users?\n",
    "+ What do the distirbutions of each feature look like?\n",
    "+ For multi-window features how do they look together?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import pyarrow as pa\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "import os\n",
    "# from collections import defaultdict\n",
    "%pylab inline\n",
    "import json\n",
    "from datetime import datetime as dt\n",
    "nat_filler = dt(1970,1,1)\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "langcodes = json.load(open('configs/ores.json','r'))['langcodes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs_dir = 'outputs'\n",
    "file_list = os.listdir(outputs_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "numericals = ['receiver_prev_edits', 'sender_prev_edits', 'sender_edits_1d_after', 'sender_edits_30d_after', 'sender_edits_90d_after', 'sender_edits_180d_after', 'receiver_edits_1d_after', 'receiver_edits_30d_after', 'receiver_edits_90d_after', 'receiver_edits_180d_after', 'receiver_prev_received', 'receiver_prev_sent', 'sender_prev_received', 'sender_prev_sent', 'receiver_thank_another_1d_after', 'receiver_thank_another_30d_after', 'receiver_thank_another_90d_after', 'receiver_thank_another_180d_after']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## Sub step Fix Enwiki file first.\n",
    "# # There may be some negative -1's because had to subtract 1 in each calculation \n",
    "# # because of inclusive indices different when useing datetime index or the standard index,\n",
    "# # but if the selection was empty we'd have a result of 0-1. \n",
    "\n",
    "# enthank_file = 'outputs/wikithank_en_20180804.csv'\n",
    "# # enthank_file = 'outputs/wikilove_en_20180805.csv'\n",
    "# endf = pd.read_csv(enthank_file)\n",
    "\n",
    "# endf.columns\n",
    "\n",
    "# numericals = ['receiver_prev_edits', 'sender_prev_edits', 'sender_edits_1d_after', 'sender_edits_30d_after', 'sender_edits_90d_after', 'sender_edits_180d_after', 'receiver_edits_1d_after', 'receiver_edits_30d_after', 'receiver_edits_90d_after', 'receiver_edits_180d_after', 'receiver_prev_received', 'receiver_prev_sent', 'sender_prev_received', 'sender_prev_sent', 'receiver_thank_another_1d_after', 'receiver_thank_another_30d_after', 'receiver_thank_another_90d_after', 'receiver_thank_another_180d_after']\n",
    "\n",
    "# def show_negs():\n",
    "#     tot = len(endf)\n",
    "#     for n in numericals:\n",
    "#         ser = endf[n]\n",
    "#         neg = ser < 0\n",
    "#         print(n)\n",
    "#         print(len(ser[neg]) / tot)\n",
    "#     #     print(ser.head(5))\n",
    "#     #     print(ser[neg].head(5))\n",
    "#         print(ser.min())\n",
    "\n",
    "# show_negs()\n",
    "\n",
    "# endf.replace(to_replace=-1, value=0, inplace=True)\n",
    "\n",
    "# show_negs()\n",
    "\n",
    "# #back enfile\n",
    "# !cp $enthank_file $enthank_file'.bak'\n",
    "# #save new enfile\n",
    "# endf.to_csv(enthank_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = dict()\n",
    "for f in file_list:\n",
    "    parts = f.split('_')\n",
    "    thanklove = parts[0].split('wiki')[1]\n",
    "    lang = parts[1]\n",
    "\n",
    "    tdf = pd.read_csv(os.path.join(outputs_dir, f), parse_dates=['timestamp', 'receiver_first_edit', 'sender_first_edit'])\n",
    "    if thanklove == 'love':\n",
    "        # i will fix this upstream so this shouldn't have to be done in the future.\n",
    "        tdf['wll_type'] = tdf['wll_type'].apply(lambda x: x.replace(\"b'\",\"\").replace(\"'\",\"\"))\n",
    "    \n",
    "    #feather requires homogenous datatypes\n",
    "    tdf['receiver'] = tdf['receiver'].astype(str)\n",
    "    tdf['sender'] = tdf['sender'].astype(str)\n",
    "    \n",
    "    tdf['thanklove'] = thanklove\n",
    "    tdf['lang'] = lang\n",
    "    files[f] = tdf\n",
    "#print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df = pd.concat(files.values())\n",
    "df = df.reset_index(drop=True)\n",
    "#df = df.fillna({'sender_first_edit':nat_filler, 'receiver_first_edit':nat_filler})\n",
    "df['probably_deleted'] = (pd.isna(df['sender_first_edit'])) | (pd.isna(df['receiver_first_edit']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "nulldf = df[df[numericals].isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1047, 34)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nulldf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in numericals:\n",
    "    df[n] = df[n].fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = {'thanklove':'gratitude',\n",
    "         'receiver_thank_another_1d_after':'receiver_gratitude_another_1d_after',\n",
    "         'receiver_thank_another_30d_after':'receiver_gratitude_another_30d_after',\n",
    "         'receiver_thank_another_90d_after':'receiver_gratitude_another_90d_after',\n",
    "         'receiver_thank_another_180d_after':'receiver_gratitude_another_180d_after',\n",
    "            }\n",
    "df.rename(mapper, axis='columns', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#redo column order\n",
    "new_order = ['lang',\n",
    "            'timestamp',\n",
    "             'gratitude',\n",
    "             'receiver',\n",
    "            'sender',\n",
    "            'receiver_id',\n",
    "            'sender_id',\n",
    "             'wll_type',\n",
    "            'probably_deleted',\n",
    "\n",
    "            'receiver_prev_edits',\n",
    "            'receiver_prev_received',\n",
    "            'receiver_prev_received_indicator',\n",
    "            'receiver_prev_sent',\n",
    "            'receiver_prev_sent_indicator',\n",
    "\n",
    "             \n",
    "            'receiver_edits_1d_after',\n",
    "            'receiver_edits_30d_after',\n",
    "            'receiver_edits_90d_after',\n",
    "            'receiver_edits_180d_after',\n",
    "\n",
    "            \n",
    "            'receiver_gratitude_another_1d_after',\n",
    "            'receiver_gratitude_another_30d_after',\n",
    "            'receiver_gratitude_another_90d_after',\n",
    "            'receiver_gratitude_another_180d_after',\n",
    "\n",
    "            'receiver_first_edit',\n",
    "            \n",
    "            'sender_prev_edits',\n",
    "            'sender_prev_received',\n",
    "            'sender_prev_received_indicator',\n",
    "            'sender_prev_sent',\n",
    "            'sender_prev_sent_indicator',\n",
    "\n",
    "            \n",
    "            'sender_edits_1d_after',\n",
    "            'sender_edits_30d_after',\n",
    "            'sender_edits_90d_after',\n",
    "            'sender_edits_180d_after',\n",
    "             \n",
    "            'sender_first_edit',\n",
    "            ]\n",
    "\n",
    "df = df[new_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "today = dt.today().strftime(\"%Y%m%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3849806"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_feather(f'transfer/gratitude_{today}.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n",
      "100000\n",
      "100000\n",
      "200000\n",
      "100000\n",
      "300000\n",
      "100000\n",
      "400000\n",
      "100000\n",
      "500000\n",
      "100000\n",
      "600000\n",
      "100000\n",
      "700000\n",
      "100000\n",
      "800000\n",
      "100000\n",
      "900000\n",
      "100000\n",
      "1000000\n",
      "100000\n",
      "1100000\n",
      "100000\n",
      "1200000\n",
      "100000\n",
      "1300000\n",
      "100000\n",
      "1400000\n",
      "100000\n",
      "1500000\n",
      "100000\n",
      "1600000\n",
      "100000\n",
      "1700000\n",
      "100000\n",
      "1800000\n",
      "100000\n",
      "1900000\n",
      "100000\n",
      "2000000\n",
      "100000\n",
      "2100000\n",
      "100000\n",
      "2200000\n",
      "100000\n",
      "2300000\n",
      "100000\n",
      "2400000\n",
      "100000\n",
      "2500000\n",
      "100000\n",
      "2600000\n",
      "100000\n",
      "2700000\n",
      "100000\n",
      "2800000\n",
      "100000\n",
      "2900000\n",
      "100000\n",
      "3000000\n",
      "100000\n",
      "3100000\n",
      "100000\n",
      "3200000\n",
      "100000\n",
      "3300000\n",
      "100000\n",
      "3400000\n",
      "100000\n",
      "3500000\n",
      "100000\n",
      "3600000\n",
      "100000\n",
      "3700000\n",
      "100000\n",
      "3800000\n",
      "49806\n",
      "3900000\n"
     ]
    }
   ],
   "source": [
    "def write_cs_rows(df, chunkrows, fname):\n",
    "    # delete file\n",
    "    try:\n",
    "        os.remove(fname)\n",
    "    except FileNotFoundError:\n",
    "        pass\n",
    "    # write header\n",
    "    df[:0].to_csv(fname, index=False)\n",
    "    # write chunks in loop with mode = a\n",
    "    required_chunks = math.ceil(len(df)/chunkrows)\n",
    "    for i in range(required_chunks):\n",
    "        startind = i*chunkrows\n",
    "        stopind = ((i+1)*chunkrows)\n",
    "        partdf = df.iloc[startind:stopind]\n",
    "        print(len(partdf))\n",
    "        partdf.to_csv(fname, mode='a', header=False)\n",
    "        gc.collect()\n",
    "        print(stopind)\n",
    "#         print(df.iloc[startind:stopind]['timestamp'].head(1))\n",
    "\n",
    "# call gc.collect()\n",
    "\n",
    "write_cs_rows(df, 100000, f'transfer/gratitude_{today}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3849806"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-0b35a516545c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdfconfirm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'transfer/gratitude_{today}.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1034\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'skipfooter not supported for iteration'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1036\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1037\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1846\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1847\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1849\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_integer_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m    809\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_integer_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    812\u001b[0m     \"\"\"\n\u001b[1;32m    813\u001b[0m     \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mprovided\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0man\u001b[0m \u001b[0minteger\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dfconfirm = pd.read_csv(f'transfer/gratitude_{today}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3849806"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfconfirm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3849803</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-30 21:47:19</td>\n",
       "      <td>thank</td>\n",
       "      <td>Pragdon</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>272157</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>20660</td>\n",
       "      <td>288</td>\n",
       "      <td>True</td>\n",
       "      <td>270</td>\n",
       "      <td>True</td>\n",
       "      <td>19</td>\n",
       "      <td>639</td>\n",
       "      <td>639</td>\n",
       "      <td>639</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-02-09 23:52:29</td>\n",
       "      <td>87764</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3173</td>\n",
       "      <td>True</td>\n",
       "      <td>18</td>\n",
       "      <td>1024</td>\n",
       "      <td>1024</td>\n",
       "      <td>1024</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3849804</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-31 19:13:07</td>\n",
       "      <td>thank</td>\n",
       "      <td>Te750iv</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>974240</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-10-16 10:56:50</td>\n",
       "      <td>87774</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3174</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "      <td>1014</td>\n",
       "      <td>1014</td>\n",
       "      <td>1014</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3849805</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-31 19:18:14</td>\n",
       "      <td>thank</td>\n",
       "      <td>Ascoltami</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>986217</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-12-12 23:08:28</td>\n",
       "      <td>87779</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3175</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "      <td>1009</td>\n",
       "      <td>1009</td>\n",
       "      <td>1009</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lang           timestamp gratitude   receiver      sender  receiver_id  sender_id wll_type  probably_deleted  receiver_prev_edits  receiver_prev_received  receiver_prev_received_indicator  receiver_prev_sent  receiver_prev_sent_indicator  receiver_edits_1d_after  receiver_edits_30d_after  receiver_edits_90d_after  receiver_edits_180d_after  receiver_gratitude_another_1d_after  receiver_gratitude_another_30d_after  receiver_gratitude_another_90d_after  receiver_gratitude_another_180d_after receiver_first_edit  sender_prev_edits  sender_prev_received  sender_prev_received_indicator  sender_prev_sent  sender_prev_sent_indicator  sender_edits_1d_after  sender_edits_30d_after  sender_edits_90d_after  sender_edits_180d_after   sender_first_edit\n",
       "3849803   tr 2018-05-30 21:47:19     thank    Pragdon  Sakhalinio       272157      86010      NaN             False                20660                     288                              True                 270                          True                       19                       639                       639                        639                                    0                                     0                                     0                                      0 2012-02-09 23:52:29              87764                  1092                            True              3173                        True                     18                    1024                    1024                     1024 2008-08-22 08:24:19\n",
       "3849804   tr 2018-05-31 19:13:07     thank    Te750iv  Sakhalinio       974240      86010      NaN             False                    4                       0                             False                   0                         False                        0                         0                         0                          0                                    0                                     0                                     0                                      0 2017-10-16 10:56:50              87774                  1092                            True              3174                        True                     11                    1014                    1014                     1014 2008-08-22 08:24:19\n",
       "3849805   tr 2018-05-31 19:18:14     thank  Ascoltami  Sakhalinio       986217      86010      NaN             False                    8                       0                             False                   0                         False                        1                         1                         1                          1                                    0                                     0                                     0                                      0 2017-12-12 23:08:28              87779                  1092                            True              3175                        True                      6                    1009                    1009                     1009 2008-08-22 08:24:19"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3849803</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-30 21:47:19</td>\n",
       "      <td>thank</td>\n",
       "      <td>Pragdon</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>272157</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>20660</td>\n",
       "      <td>288</td>\n",
       "      <td>True</td>\n",
       "      <td>270</td>\n",
       "      <td>True</td>\n",
       "      <td>19</td>\n",
       "      <td>639</td>\n",
       "      <td>639</td>\n",
       "      <td>639</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-02-09 23:52:29</td>\n",
       "      <td>87764</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3173</td>\n",
       "      <td>True</td>\n",
       "      <td>18</td>\n",
       "      <td>1024</td>\n",
       "      <td>1024</td>\n",
       "      <td>1024</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3849804</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-31 19:13:07</td>\n",
       "      <td>thank</td>\n",
       "      <td>Te750iv</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>974240</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-10-16 10:56:50</td>\n",
       "      <td>87774</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3174</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "      <td>1014</td>\n",
       "      <td>1014</td>\n",
       "      <td>1014</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3849805</th>\n",
       "      <td>tr</td>\n",
       "      <td>2018-05-31 19:18:14</td>\n",
       "      <td>thank</td>\n",
       "      <td>Ascoltami</td>\n",
       "      <td>Sakhalinio</td>\n",
       "      <td>986217</td>\n",
       "      <td>86010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-12-12 23:08:28</td>\n",
       "      <td>87779</td>\n",
       "      <td>1092</td>\n",
       "      <td>True</td>\n",
       "      <td>3175</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "      <td>1009</td>\n",
       "      <td>1009</td>\n",
       "      <td>1009</td>\n",
       "      <td>2008-08-22 08:24:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lang            timestamp gratitude   receiver      sender  receiver_id  sender_id wll_type  probably_deleted  receiver_prev_edits  receiver_prev_received  receiver_prev_received_indicator  receiver_prev_sent  receiver_prev_sent_indicator  receiver_edits_1d_after  receiver_edits_30d_after  receiver_edits_90d_after  receiver_edits_180d_after  receiver_gratitude_another_1d_after  receiver_gratitude_another_30d_after  receiver_gratitude_another_90d_after  receiver_gratitude_another_180d_after  receiver_first_edit  sender_prev_edits  sender_prev_received  sender_prev_received_indicator  sender_prev_sent  sender_prev_sent_indicator  sender_edits_1d_after  sender_edits_30d_after  sender_edits_90d_after  sender_edits_180d_after    sender_first_edit\n",
       "3849803   tr  2018-05-30 21:47:19     thank    Pragdon  Sakhalinio       272157      86010      NaN             False                20660                     288                              True                 270                          True                       19                       639                       639                        639                                    0                                     0                                     0                                      0  2012-02-09 23:52:29              87764                  1092                            True              3173                        True                     18                    1024                    1024                     1024  2008-08-22 08:24:19\n",
       "3849804   tr  2018-05-31 19:13:07     thank    Te750iv  Sakhalinio       974240      86010      NaN             False                    4                       0                             False                   0                         False                        0                         0                         0                          0                                    0                                     0                                     0                                      0  2017-10-16 10:56:50              87774                  1092                            True              3174                        True                     11                    1014                    1014                     1014  2008-08-22 08:24:19\n",
       "3849805   tr  2018-05-31 19:18:14     thank  Ascoltami  Sakhalinio       986217      86010      NaN             False                    8                       0                             False                   0                         False                        1                         1                         1                          1                                    0                                     0                                     0                                      0  2017-12-12 23:08:28              87779                  1092                            True              3175                        True                      6                    1009                    1009                     1009  2008-08-22 08:24:19"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfconfirm.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99998                     Milez189\n",
       "99999                    Henrik.01\n",
       "100000                Vlad2000Plus\n",
       "100001    Vbngrbsknhobnpgrnbongonj\n",
       "100002                   Maas Leag\n",
       "Name: receiver, dtype: object"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[99998:100003]['receiver']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>ru</td>\n",
       "      <td>2016-02-28 20:39:15</td>\n",
       "      <td>thank</td>\n",
       "      <td>Milez189</td>\n",
       "      <td>Roundabout</td>\n",
       "      <td>1290702</td>\n",
       "      <td>429898</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>39674</td>\n",
       "      <td>290</td>\n",
       "      <td>True</td>\n",
       "      <td>97</td>\n",
       "      <td>True</td>\n",
       "      <td>153</td>\n",
       "      <td>6253</td>\n",
       "      <td>16153</td>\n",
       "      <td>16168</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>2014-07-14 10:46:21</td>\n",
       "      <td>8780</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "      <td>203</td>\n",
       "      <td>487</td>\n",
       "      <td>724</td>\n",
       "      <td>2010-02-11 22:08:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>ru</td>\n",
       "      <td>2016-02-28 21:03:47</td>\n",
       "      <td>thank</td>\n",
       "      <td>Henrik.01</td>\n",
       "      <td>Mahno30</td>\n",
       "      <td>1164403</td>\n",
       "      <td>1864455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-03-22 06:54:10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>ru</td>\n",
       "      <td>2016-02-28 21:03:58</td>\n",
       "      <td>thank</td>\n",
       "      <td>Vlad2000Plus</td>\n",
       "      <td>KartMan</td>\n",
       "      <td>8414</td>\n",
       "      <td>1864434</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>11888</td>\n",
       "      <td>53</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>29</td>\n",
       "      <td>1671</td>\n",
       "      <td>1787</td>\n",
       "      <td>1991</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2006-01-03 01:01:44</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100001</th>\n",
       "      <td>ru</td>\n",
       "      <td>2016-02-28 21:12:37</td>\n",
       "      <td>thank</td>\n",
       "      <td>Vbngrbsknhobnpgrnbongonj</td>\n",
       "      <td>Повелитель Звёзд</td>\n",
       "      <td>1295384</td>\n",
       "      <td>1346505</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-09 18:15:52</td>\n",
       "      <td>2477</td>\n",
       "      <td>24</td>\n",
       "      <td>True</td>\n",
       "      <td>861</td>\n",
       "      <td>True</td>\n",
       "      <td>23</td>\n",
       "      <td>486</td>\n",
       "      <td>1489</td>\n",
       "      <td>2538</td>\n",
       "      <td>2014-09-03 00:06:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100002</th>\n",
       "      <td>ru</td>\n",
       "      <td>2016-02-28 21:20:19</td>\n",
       "      <td>thank</td>\n",
       "      <td>Maas Leag</td>\n",
       "      <td>Anji-dag</td>\n",
       "      <td>1775212</td>\n",
       "      <td>1144114</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>190</td>\n",
       "      <td>191</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-10-24 07:11:06</td>\n",
       "      <td>470</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>60</td>\n",
       "      <td>64</td>\n",
       "      <td>2013-11-12 17:23:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       lang            timestamp gratitude                  receiver            sender  receiver_id  sender_id wll_type  probably_deleted  receiver_prev_edits  receiver_prev_received  receiver_prev_received_indicator  receiver_prev_sent  receiver_prev_sent_indicator  receiver_edits_1d_after  receiver_edits_30d_after  receiver_edits_90d_after  receiver_edits_180d_after  receiver_gratitude_another_1d_after  receiver_gratitude_another_30d_after  receiver_gratitude_another_90d_after  receiver_gratitude_another_180d_after  receiver_first_edit  sender_prev_edits  sender_prev_received  sender_prev_received_indicator  sender_prev_sent  sender_prev_sent_indicator  sender_edits_1d_after  sender_edits_30d_after  sender_edits_90d_after  sender_edits_180d_after    sender_first_edit\n",
       "99998    ru  2016-02-28 20:39:15     thank                  Milez189        Roundabout      1290702     429898      NaN             False                39674                     290                              True                  97                          True                      153                      6253                     16153                      16168                                    0                                     1                                    18                                     18  2014-07-14 10:46:21               8780                    18                            True                40                        True                      9                     203                     487                      724  2010-02-11 22:08:08\n",
       "99999    ru  2016-02-28 21:03:47     thank                 Henrik.01           Mahno30      1164403    1864455      NaN              True                    2                       0                             False                   0                         False                        0                         0                         0                          0                                    0                                     0                                     0                                      0  2015-03-22 06:54:10                  0                     0                           False                 0                       False                      0                       0                       0                        0                  NaN\n",
       "100000   ru  2016-02-28 21:03:58     thank              Vlad2000Plus           KartMan         8414    1864434      NaN              True                11888                      53                              True                   0                         False                       29                      1671                      1787                       1991                                    0                                     1                                     1                                      1  2006-01-03 01:01:44                  0                     0                           False                 0                       False                      0                       0                       0                        0                  NaN\n",
       "100001   ru  2016-02-28 21:12:37     thank  Vbngrbsknhobnpgrnbongonj  Повелитель Звёзд      1295384    1346505      NaN             False                   31                       0                             False                   0                         False                        0                         2                         3                          3                                    0                                     0                                     0                                      0  2014-08-09 18:15:52               2477                    24                            True               861                        True                     23                     486                    1489                     2538  2014-09-03 00:06:21\n",
       "100002   ru  2016-02-28 21:20:19     thank                 Maas Leag          Anji-dag      1775212    1144114      NaN             False                   51                       0                             False                   0                         False                        0                        11                       190                        191                                    0                                     0                                     0                                      0  2015-10-24 07:11:06                470                     0                           False                14                        True                      0                       7                      60                       64  2013-11-12 17:23:41"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfconfirm[99998:100003]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lang,timestamp,gratitude,receiver,sender,receiver_id,sender_id,wll_type,probably_deleted,receiver_prev_edits,receiver_prev_received,receiver_prev_received_indicator,receiver_prev_sent,receiver_prev_sent_indicator,receiver_edits_1d_after,receiver_edits_30d_after,receiver_edits_90d_after,receiver_edits_180d_after,receiver_gratitude_another_1d_after,receiver_gratitude_another_30d_after,receiver_gratitude_another_90d_after,receiver_gratitude_another_180d_after,receiver_first_edit,sender_prev_edits,sender_prev_received,sender_prev_received_indicator,sender_prev_sent,sender_prev_sent_indicator,sender_edits_1d_after,sender_edits_30d_after,sender_edits_90d_after,sender_edits_180d_after,sender_first_edit\r\n",
      "0,ar,2011-10-12 20:38:38,love,341772,341772,341772,341772,barnstar-original,False,2,0,False,0,False,0,0,0,1,0,0,0,0,2011-10-12 20:35:46,2,0,False,0,False,0,0,0,1,2011-10-12 20:35:46\r\n",
      "1,ar,2011-10-13 11:56:45,love,418003,418003,418003,418003,kitten,False,0,0,False,0,False,1,1,1,1,0,0,0,0,2011-10-13 11:56:46,0,0,False,0,False,1,1,1,1,2011-10-13 11:56:46\r\n",
      "2,ar,2011-10-13 12:19:32,love,411973,411973,411973,411973,barnstar-original,False,1,0,False,0,False,1,1,1,1,1,1,1,1,2011-10-13 12:14:15,1,0,False,0,False,1,1,1,1,2011-10-13 12:14:15\r\n",
      "3,ar,2011-10-13 12:19:47,love,411973,411973,411973,411973,barnstar-original,False,2,1,True,1,True,1,1,1,1,0,0,0,0,2011-10-13 12:14:15,2,1,True,1,True,1,1,1,1,2011-10-13 12:14:15\r\n",
      "4,ar,2011-10-13 13:26:24,love,334655,334655,334655,334655,barnstar-original,False,437,0,False,0,False,4,38,91,91,1,1,1,1,2010-12-11 18:40:48,437,0,False,0,False,4,38,91,91,2010-12-11 18:40:48\r\n",
      "5,ar,2011-10-13 13:33:02,love,9793,334655,9793,334655,barnstar-original,False,35135,0,False,0,False,6,511,1420,3357,0,0,0,0,2006-04-01 07:22:10,438,1,True,1,True,3,37,90,90,2010-12-11 18:40:48\r\n",
      "6,ar,2011-10-13 14:19:27,love,333042,333042,333042,333042,barnstar-original,False,0,0,False,0,False,4,4,4,4,0,0,0,0,2011-10-13 14:19:28,0,0,False,0,False,4,4,4,4,2011-10-13 14:19:28\r\n",
      "7,ar,2011-10-13 20:48:06,love,418190,418190,418190,418190,barnstar-diligence,False,0,0,False,0,False,1,1,1,1,0,0,0,0,2011-10-13 20:48:07,0,0,False,0,False,1,1,1,1,2011-10-13 20:48:07\r\n",
      "8,ar,2011-10-14 05:12:22,love,157430,418293,157430,418293,barnstar-original,False,43960,0,False,0,False,334,2135,5579,7852,0,1,1,1,2008-11-18 23:40:47,5,0,False,0,False,83,1895,3622,3638,2011-10-14 04:54:36\r\n"
     ]
    }
   ],
   "source": [
    "!head 'transfer/gratitude_20180813.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [lang, timestamp, gratitude, receiver, sender, receiver_id, sender_id, wll_type, probably_deleted, receiver_prev_edits, receiver_prev_received, receiver_prev_received_indicator, receiver_prev_sent, receiver_prev_sent_indicator, receiver_edits_1d_after, receiver_edits_30d_after, receiver_edits_90d_after, receiver_edits_180d_after, receiver_gratitude_another_1d_after, receiver_gratitude_another_30d_after, receiver_gratitude_another_90d_after, receiver_gratitude_another_180d_after, receiver_first_edit, sender_prev_edits, sender_prev_received, sender_prev_received_indicator, sender_prev_sent, sender_prev_sent_indicator, sender_edits_1d_after, sender_edits_30d_after, sender_edits_90d_after, sender_edits_180d_after, sender_first_edit]\n",
       "Index: []"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ar</td>\n",
       "      <td>2011-10-12 20:38:38</td>\n",
       "      <td>love</td>\n",
       "      <td>341772</td>\n",
       "      <td>341772</td>\n",
       "      <td>341772</td>\n",
       "      <td>341772</td>\n",
       "      <td>barnstar-original</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2011-10-12 20:35:46</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2011-10-12 20:35:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ar</td>\n",
       "      <td>2011-10-13 11:56:45</td>\n",
       "      <td>love</td>\n",
       "      <td>418003</td>\n",
       "      <td>418003</td>\n",
       "      <td>418003</td>\n",
       "      <td>418003</td>\n",
       "      <td>kitten</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2011-10-13 11:56:46</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2011-10-13 11:56:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ar</td>\n",
       "      <td>2011-10-13 12:19:32</td>\n",
       "      <td>love</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>barnstar-original</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2011-10-13 12:14:15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2011-10-13 12:14:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ar</td>\n",
       "      <td>2011-10-13 12:19:47</td>\n",
       "      <td>love</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>411973</td>\n",
       "      <td>barnstar-original</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2011-10-13 12:14:15</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2011-10-13 12:14:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ar</td>\n",
       "      <td>2011-10-13 13:26:24</td>\n",
       "      <td>love</td>\n",
       "      <td>334655</td>\n",
       "      <td>334655</td>\n",
       "      <td>334655</td>\n",
       "      <td>334655</td>\n",
       "      <td>barnstar-original</td>\n",
       "      <td>False</td>\n",
       "      <td>437</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>91</td>\n",
       "      <td>91</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2010-12-11 18:40:48</td>\n",
       "      <td>437</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>91</td>\n",
       "      <td>91</td>\n",
       "      <td>2010-12-11 18:40:48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang           timestamp gratitude receiver  sender  receiver_id  sender_id           wll_type  probably_deleted  receiver_prev_edits  receiver_prev_received  receiver_prev_received_indicator  receiver_prev_sent  receiver_prev_sent_indicator  receiver_edits_1d_after  receiver_edits_30d_after  receiver_edits_90d_after  receiver_edits_180d_after  receiver_gratitude_another_1d_after  receiver_gratitude_another_30d_after  receiver_gratitude_another_90d_after  receiver_gratitude_another_180d_after receiver_first_edit  sender_prev_edits  sender_prev_received  sender_prev_received_indicator  sender_prev_sent  sender_prev_sent_indicator  sender_edits_1d_after  sender_edits_30d_after  sender_edits_90d_after  sender_edits_180d_after   sender_first_edit\n",
       "0   ar 2011-10-12 20:38:38      love   341772  341772       341772     341772  barnstar-original             False                    2                       0                             False                   0                         False                        0                         0                         0                          1                                    0                                     0                                     0                                      0 2011-10-12 20:35:46                  2                     0                           False                 0                       False                      0                       0                       0                        1 2011-10-12 20:35:46\n",
       "1   ar 2011-10-13 11:56:45      love   418003  418003       418003     418003             kitten             False                    0                       0                             False                   0                         False                        1                         1                         1                          1                                    0                                     0                                     0                                      0 2011-10-13 11:56:46                  0                     0                           False                 0                       False                      1                       1                       1                        1 2011-10-13 11:56:46\n",
       "2   ar 2011-10-13 12:19:32      love   411973  411973       411973     411973  barnstar-original             False                    1                       0                             False                   0                         False                        1                         1                         1                          1                                    1                                     1                                     1                                      1 2011-10-13 12:14:15                  1                     0                           False                 0                       False                      1                       1                       1                        1 2011-10-13 12:14:15\n",
       "3   ar 2011-10-13 12:19:47      love   411973  411973       411973     411973  barnstar-original             False                    2                       1                              True                   1                          True                        1                         1                         1                          1                                    0                                     0                                     0                                      0 2011-10-13 12:14:15                  2                     1                            True                 1                        True                      1                       1                       1                        1 2011-10-13 12:14:15\n",
       "4   ar 2011-10-13 13:26:24      love   334655  334655       334655     334655  barnstar-original             False                  437                       0                             False                   0                         False                        4                        38                        91                         91                                    1                                     1                                     1                                      1 2010-12-11 18:40:48                437                     0                           False                 0                       False                      4                      38                      91                       91 2010-12-11 18:40:48"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dfconfirm' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-e125a3f74576>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mdel\u001b[0m \u001b[0mdfconfirm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'dfconfirm' is not defined"
     ]
    }
   ],
   "source": [
    "del dfconfirm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[Errno 12] Cannot allocate memory",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-71f111ff433e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'du -h transfer/*'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36msystem_piped\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m   2209\u001b[0m         \u001b[0;31m# a non-None value would trigger :func:`sys.displayhook` calls.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2210\u001b[0m         \u001b[0;31m# Instead, we store the exit_code in user_ns.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2211\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_ns\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'_exit_code'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msystem_raw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/utils/_process_posix.py\u001b[0m in \u001b[0;36msystem\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m    155\u001b[0m                 \u001b[0mchild\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpexpect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspawnb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'-c'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmd\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Pexpect-U\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m                 \u001b[0mchild\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpexpect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspawn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'-c'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmd\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Vanilla Pexpect\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m             \u001b[0mflush\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pexpect/pty_spawn.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, command, args, timeout, maxread, searchwindowsize, logfile, cwd, env, ignore_sighup, echo, preexec_fn, encoding, codec_errors, dimensions)\u001b[0m\n\u001b[1;32m    195\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'<pexpect factory incomplete>'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spawn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreexec_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdimensions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__str__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pexpect/pty_spawn.py\u001b[0m in \u001b[0;36m_spawn\u001b[0;34m(self, command, args, preexec_fn, dimensions)\u001b[0m\n\u001b[1;32m    295\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m         self.ptyproc = self._spawnpty(self.args, env=self.env,\n\u001b[0;32m--> 297\u001b[0;31m                                      cwd=self.cwd, **kwargs)\n\u001b[0m\u001b[1;32m    298\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mptyproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pexpect/pty_spawn.py\u001b[0m in \u001b[0;36m_spawnpty\u001b[0;34m(self, args, **kwargs)\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_spawnpty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m         \u001b[0;34m'''Spawn a pty and return an instance of PtyProcess.'''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mptyprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPtyProcess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspawn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/ptyprocess/ptyprocess.py\u001b[0m in \u001b[0;36mspawn\u001b[0;34m(cls, argv, cwd, env, echo, preexec_fn, dimensions)\u001b[0m\n\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0muse_native_pty_fork\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m             \u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    227\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    228\u001b[0m             \u001b[0;31m# Use internal fork_pty, for Solaris\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/pty.py\u001b[0m in \u001b[0;36mfork\u001b[0;34m()\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0mmaster_fd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslave_fd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopenpty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m     \u001b[0mpid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mCHILD\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m         \u001b[0;31m# Establish a new session.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [Errno 12] Cannot allocate memory"
     ]
    }
   ],
   "source": [
    "!du -h transfer/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gratitude</th>\n",
       "      <th>receiver</th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver_id</th>\n",
       "      <th>sender_id</th>\n",
       "      <th>wll_type</th>\n",
       "      <th>probably_deleted</th>\n",
       "      <th>receiver_prev_edits</th>\n",
       "      <th>receiver_prev_received</th>\n",
       "      <th>receiver_prev_received_indicator</th>\n",
       "      <th>receiver_prev_sent</th>\n",
       "      <th>receiver_prev_sent_indicator</th>\n",
       "      <th>receiver_edits_1d_after</th>\n",
       "      <th>receiver_edits_30d_after</th>\n",
       "      <th>receiver_edits_90d_after</th>\n",
       "      <th>receiver_edits_180d_after</th>\n",
       "      <th>receiver_gratitude_another_1d_after</th>\n",
       "      <th>receiver_gratitude_another_30d_after</th>\n",
       "      <th>receiver_gratitude_another_90d_after</th>\n",
       "      <th>receiver_gratitude_another_180d_after</th>\n",
       "      <th>receiver_first_edit</th>\n",
       "      <th>sender_prev_edits</th>\n",
       "      <th>sender_prev_received</th>\n",
       "      <th>sender_prev_received_indicator</th>\n",
       "      <th>sender_prev_sent</th>\n",
       "      <th>sender_prev_sent_indicator</th>\n",
       "      <th>sender_edits_1d_after</th>\n",
       "      <th>sender_edits_30d_after</th>\n",
       "      <th>sender_edits_90d_after</th>\n",
       "      <th>sender_edits_180d_after</th>\n",
       "      <th>sender_first_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3580841</th>\n",
       "      <td>en</td>\n",
       "      <td>2018-05-31 23:56:04</td>\n",
       "      <td>thank</td>\n",
       "      <td>Dilidor</td>\n",
       "      <td>Toughpigs</td>\n",
       "      <td>25648301</td>\n",
       "      <td>578047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>8404</td>\n",
       "      <td>76</td>\n",
       "      <td>True</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>20</td>\n",
       "      <td>261</td>\n",
       "      <td>572</td>\n",
       "      <td>572</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-06-30 19:18:29</td>\n",
       "      <td>7571</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>72</td>\n",
       "      <td>149</td>\n",
       "      <td>171</td>\n",
       "      <td>171</td>\n",
       "      <td>2005-11-14 01:46:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3580842</th>\n",
       "      <td>en</td>\n",
       "      <td>2018-05-31 23:56:22</td>\n",
       "      <td>thank</td>\n",
       "      <td>Kyteto</td>\n",
       "      <td>Ahunt</td>\n",
       "      <td>9218062</td>\n",
       "      <td>253891</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>22861</td>\n",
       "      <td>125</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>288</td>\n",
       "      <td>354</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2009-03-15 01:40:45</td>\n",
       "      <td>145639</td>\n",
       "      <td>2029</td>\n",
       "      <td>True</td>\n",
       "      <td>7348</td>\n",
       "      <td>True</td>\n",
       "      <td>48</td>\n",
       "      <td>902</td>\n",
       "      <td>1131</td>\n",
       "      <td>1131</td>\n",
       "      <td>2005-04-30 21:23:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3580843</th>\n",
       "      <td>en</td>\n",
       "      <td>2018-05-31 23:56:35</td>\n",
       "      <td>thank</td>\n",
       "      <td>Haploidavey</td>\n",
       "      <td>InformationvsInjustice</td>\n",
       "      <td>8882122</td>\n",
       "      <td>16692328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>33084</td>\n",
       "      <td>166</td>\n",
       "      <td>True</td>\n",
       "      <td>94</td>\n",
       "      <td>True</td>\n",
       "      <td>24</td>\n",
       "      <td>335</td>\n",
       "      <td>508</td>\n",
       "      <td>508</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2009-02-02 19:35:30</td>\n",
       "      <td>2748</td>\n",
       "      <td>13</td>\n",
       "      <td>True</td>\n",
       "      <td>297</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>134</td>\n",
       "      <td>192</td>\n",
       "      <td>192</td>\n",
       "      <td>2012-04-22 03:13:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3580844</th>\n",
       "      <td>en</td>\n",
       "      <td>2018-05-31 23:57:10</td>\n",
       "      <td>thank</td>\n",
       "      <td>Acroterion</td>\n",
       "      <td>InformationvsInjustice</td>\n",
       "      <td>1839637</td>\n",
       "      <td>16692328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>146071</td>\n",
       "      <td>1511</td>\n",
       "      <td>True</td>\n",
       "      <td>217</td>\n",
       "      <td>True</td>\n",
       "      <td>36</td>\n",
       "      <td>991</td>\n",
       "      <td>1409</td>\n",
       "      <td>1409</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2006-07-22 01:31:04</td>\n",
       "      <td>2748</td>\n",
       "      <td>13</td>\n",
       "      <td>True</td>\n",
       "      <td>298</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>134</td>\n",
       "      <td>192</td>\n",
       "      <td>192</td>\n",
       "      <td>2012-04-22 03:13:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3580845</th>\n",
       "      <td>en</td>\n",
       "      <td>2018-05-31 23:57:44</td>\n",
       "      <td>thank</td>\n",
       "      <td>Pegship</td>\n",
       "      <td>Jjjjjjdddddd</td>\n",
       "      <td>355698</td>\n",
       "      <td>20083999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>93754</td>\n",
       "      <td>88</td>\n",
       "      <td>True</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>119</td>\n",
       "      <td>146</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2005-07-29 19:29:33</td>\n",
       "      <td>4389</td>\n",
       "      <td>48</td>\n",
       "      <td>True</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>2013-11-06 04:54:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lang           timestamp gratitude     receiver                  sender  receiver_id  sender_id wll_type  probably_deleted  receiver_prev_edits  receiver_prev_received  receiver_prev_received_indicator  receiver_prev_sent  receiver_prev_sent_indicator  receiver_edits_1d_after  receiver_edits_30d_after  receiver_edits_90d_after  receiver_edits_180d_after  receiver_gratitude_another_1d_after  receiver_gratitude_another_30d_after  receiver_gratitude_another_90d_after  receiver_gratitude_another_180d_after receiver_first_edit  sender_prev_edits  sender_prev_received  sender_prev_received_indicator  sender_prev_sent  sender_prev_sent_indicator  sender_edits_1d_after  sender_edits_30d_after  sender_edits_90d_after  sender_edits_180d_after   sender_first_edit\n",
       "3580841   en 2018-05-31 23:56:04     thank      Dilidor               Toughpigs     25648301     578047      NaN             False                 8404                      76                              True                  17                          True                       20                       261                       572                        572                                    0                                     0                                     0                                      0 2015-06-30 19:18:29               7571                    35                            True                 5                        True                     72                     149                     171                      171 2005-11-14 01:46:31\n",
       "3580842   en 2018-05-31 23:56:22     thank       Kyteto                   Ahunt      9218062     253891      NaN             False                22861                     125                              True                   0                         False                        1                       288                       354                        354                                    0                                     0                                     0                                      0 2009-03-15 01:40:45             145639                  2029                            True              7348                        True                     48                     902                    1131                     1131 2005-04-30 21:23:09\n",
       "3580843   en 2018-05-31 23:56:35     thank  Haploidavey  InformationvsInjustice      8882122   16692328      NaN             False                33084                     166                              True                  94                          True                       24                       335                       508                        508                                    0                                     0                                     0                                      0 2009-02-02 19:35:30               2748                    13                            True               297                        True                      3                     134                     192                      192 2012-04-22 03:13:04\n",
       "3580844   en 2018-05-31 23:57:10     thank   Acroterion  InformationvsInjustice      1839637   16692328      NaN             False               146071                    1511                              True                 217                          True                       36                       991                      1409                       1409                                    0                                     0                                     0                                      0 2006-07-22 01:31:04               2748                    13                            True               298                        True                      3                     134                     192                      192 2012-04-22 03:13:04\n",
       "3580845   en 2018-05-31 23:57:44     thank      Pegship            Jjjjjjdddddd       355698   20083999      NaN             False                93754                      88                              True                  35                          True                        1                       119                       146                        146                                    0                                     0                                     0                                      0 2005-07-29 19:29:33               4389                    48                            True                80                        True                      5                      58                      58                       58 2013-11-06 04:54:04"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['lang']=='en'].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frac_nan(df):\n",
    "    sender_nan = df[pd.isna(df['sender_first_edit'])]\n",
    "    receiver_nan = df[pd.isna(df['receiver_first_edit'])]\n",
    "    either_nan = df[(pd.isna(df['sender_first_edit'])) | (pd.isna(df['receiver_first_edit']))]\n",
    "    denom = len(df)\n",
    "    sender_nan_frac = len(sender_nan) / denom\n",
    "    receiver_nan_frac = len(receiver_nan) / denom\n",
    "    either_nan_frac = len(either_nan) / denom\n",
    "    return {'sender_nan_frac': sender_nan_frac,\n",
    "            'receiver_nan_frac': receiver_nan_frac,\n",
    "            'either_nan_frac': either_nan_frac}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'either_nan_frac': 0.012047360308545418,\n",
       " 'receiver_nan_frac': 0.0025432450362433846,\n",
       " 'sender_nan_frac': 0.009862315140035627}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frac_nan(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_fracs = {}\n",
    "for fname, df in files.items():\n",
    "    nan_fracs[fname] = frac_nan(df)\n",
    "nan_frac_df = pd.DataFrame.from_dict(nan_fracs, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sender_nan_frac</th>\n",
       "      <th>receiver_nan_frac</th>\n",
       "      <th>either_nan_frac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wikithank_he_20180627.csv</th>\n",
       "      <td>0.001004</td>\n",
       "      <td>0.000854</td>\n",
       "      <td>0.001858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_de_20180712.csv</th>\n",
       "      <td>0.002218</td>\n",
       "      <td>0.000606</td>\n",
       "      <td>0.002805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_et_20180704.csv</th>\n",
       "      <td>0.002881</td>\n",
       "      <td>0.000206</td>\n",
       "      <td>0.003087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_no_20180703.csv</th>\n",
       "      <td>0.002247</td>\n",
       "      <td>0.001324</td>\n",
       "      <td>0.003170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_hu_20180627.csv</th>\n",
       "      <td>0.003014</td>\n",
       "      <td>0.000328</td>\n",
       "      <td>0.003343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_sv_20180628.csv</th>\n",
       "      <td>0.003561</td>\n",
       "      <td>0.000838</td>\n",
       "      <td>0.004150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_uk_20180703.csv</th>\n",
       "      <td>0.001439</td>\n",
       "      <td>0.002878</td>\n",
       "      <td>0.004317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_cs_20180704.csv</th>\n",
       "      <td>0.003793</td>\n",
       "      <td>0.000722</td>\n",
       "      <td>0.004392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_hr_20180702.csv</th>\n",
       "      <td>0.004275</td>\n",
       "      <td>0.000570</td>\n",
       "      <td>0.004845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_uk_20180703.csv</th>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.000715</td>\n",
       "      <td>0.004860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_el_20180703.csv</th>\n",
       "      <td>0.004713</td>\n",
       "      <td>0.000973</td>\n",
       "      <td>0.005686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_nl_20180704.csv</th>\n",
       "      <td>0.005212</td>\n",
       "      <td>0.001593</td>\n",
       "      <td>0.006669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_fr_20180707.csv</th>\n",
       "      <td>0.005588</td>\n",
       "      <td>0.001508</td>\n",
       "      <td>0.006933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_he_20180626.csv</th>\n",
       "      <td>0.005771</td>\n",
       "      <td>0.003463</td>\n",
       "      <td>0.008310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_bn_20180702.csv</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008955</td>\n",
       "      <td>0.008955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_fi_20180704.csv</th>\n",
       "      <td>0.008832</td>\n",
       "      <td>0.000652</td>\n",
       "      <td>0.009411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_it_20180703.csv</th>\n",
       "      <td>0.008849</td>\n",
       "      <td>0.001868</td>\n",
       "      <td>0.010672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_en_20180804.csv</th>\n",
       "      <td>0.009936</td>\n",
       "      <td>0.002117</td>\n",
       "      <td>0.011867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_fa_20180628.csv</th>\n",
       "      <td>0.004096</td>\n",
       "      <td>0.011659</td>\n",
       "      <td>0.013707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_fa_20180629.csv</th>\n",
       "      <td>0.013024</td>\n",
       "      <td>0.003540</td>\n",
       "      <td>0.015916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_es_20180626.csv</th>\n",
       "      <td>0.013939</td>\n",
       "      <td>0.003094</td>\n",
       "      <td>0.016594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_ru_20180705.csv</th>\n",
       "      <td>0.015085</td>\n",
       "      <td>0.003830</td>\n",
       "      <td>0.018767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_no_20180703.csv</th>\n",
       "      <td>0.012723</td>\n",
       "      <td>0.012723</td>\n",
       "      <td>0.019084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_pt_20180627.csv</th>\n",
       "      <td>0.017571</td>\n",
       "      <td>0.002614</td>\n",
       "      <td>0.019997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_hu_20180627.csv</th>\n",
       "      <td>0.015789</td>\n",
       "      <td>0.013158</td>\n",
       "      <td>0.022368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_ko_20180703.csv</th>\n",
       "      <td>0.015359</td>\n",
       "      <td>0.008678</td>\n",
       "      <td>0.023429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_ar_20180623.csv</th>\n",
       "      <td>0.008521</td>\n",
       "      <td>0.018583</td>\n",
       "      <td>0.024101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_ko_20180703.csv</th>\n",
       "      <td>0.009604</td>\n",
       "      <td>0.019208</td>\n",
       "      <td>0.024610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_es_20180625.csv</th>\n",
       "      <td>0.018370</td>\n",
       "      <td>0.008183</td>\n",
       "      <td>0.025718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_en_20180713.csv</th>\n",
       "      <td>0.013531</td>\n",
       "      <td>0.017264</td>\n",
       "      <td>0.027140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_ar_20180623.csv</th>\n",
       "      <td>0.023409</td>\n",
       "      <td>0.004375</td>\n",
       "      <td>0.027191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_tr_20180628.csv</th>\n",
       "      <td>0.026359</td>\n",
       "      <td>0.003040</td>\n",
       "      <td>0.028326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_ro_20180704.csv</th>\n",
       "      <td>0.029215</td>\n",
       "      <td>0.000651</td>\n",
       "      <td>0.029773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_vi_20180703.csv</th>\n",
       "      <td>0.011256</td>\n",
       "      <td>0.020142</td>\n",
       "      <td>0.031398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_bn_20180702.csv</th>\n",
       "      <td>0.030090</td>\n",
       "      <td>0.002760</td>\n",
       "      <td>0.032850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_pl_20180704.csv</th>\n",
       "      <td>0.038638</td>\n",
       "      <td>0.001087</td>\n",
       "      <td>0.039515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_id_20180703.csv</th>\n",
       "      <td>0.043183</td>\n",
       "      <td>0.006513</td>\n",
       "      <td>0.047912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikithank_vi_20180703.csv</th>\n",
       "      <td>0.047970</td>\n",
       "      <td>0.003739</td>\n",
       "      <td>0.051282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_tr_20180628.csv</th>\n",
       "      <td>0.036665</td>\n",
       "      <td>0.022064</td>\n",
       "      <td>0.056132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_pt_20180627.csv</th>\n",
       "      <td>0.059546</td>\n",
       "      <td>0.037672</td>\n",
       "      <td>0.070213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikilove_sv_20180627.csv</th>\n",
       "      <td>0.133505</td>\n",
       "      <td>0.003093</td>\n",
       "      <td>0.135567</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           sender_nan_frac  receiver_nan_frac  either_nan_frac\n",
       "wikithank_he_20180627.csv         0.001004           0.000854         0.001858\n",
       "wikithank_de_20180712.csv         0.002218           0.000606         0.002805\n",
       "wikithank_et_20180704.csv         0.002881           0.000206         0.003087\n",
       "wikithank_no_20180703.csv         0.002247           0.001324         0.003170\n",
       "wikithank_hu_20180627.csv         0.003014           0.000328         0.003343\n",
       "wikithank_sv_20180628.csv         0.003561           0.000838         0.004150\n",
       "wikilove_uk_20180703.csv          0.001439           0.002878         0.004317\n",
       "wikithank_cs_20180704.csv         0.003793           0.000722         0.004392\n",
       "wikithank_hr_20180702.csv         0.004275           0.000570         0.004845\n",
       "wikithank_uk_20180703.csv         0.004158           0.000715         0.004860\n",
       "wikithank_el_20180703.csv         0.004713           0.000973         0.005686\n",
       "wikithank_nl_20180704.csv         0.005212           0.001593         0.006669\n",
       "wikithank_fr_20180707.csv         0.005588           0.001508         0.006933\n",
       "wikilove_he_20180626.csv          0.005771           0.003463         0.008310\n",
       "wikilove_bn_20180702.csv          0.000000           0.008955         0.008955\n",
       "wikithank_fi_20180704.csv         0.008832           0.000652         0.009411\n",
       "wikithank_it_20180703.csv         0.008849           0.001868         0.010672\n",
       "wikithank_en_20180804.csv         0.009936           0.002117         0.011867\n",
       "wikilove_fa_20180628.csv          0.004096           0.011659         0.013707\n",
       "wikithank_fa_20180629.csv         0.013024           0.003540         0.015916\n",
       "wikithank_es_20180626.csv         0.013939           0.003094         0.016594\n",
       "wikithank_ru_20180705.csv         0.015085           0.003830         0.018767\n",
       "wikilove_no_20180703.csv          0.012723           0.012723         0.019084\n",
       "wikithank_pt_20180627.csv         0.017571           0.002614         0.019997\n",
       "wikilove_hu_20180627.csv          0.015789           0.013158         0.022368\n",
       "wikithank_ko_20180703.csv         0.015359           0.008678         0.023429\n",
       "wikilove_ar_20180623.csv          0.008521           0.018583         0.024101\n",
       "wikilove_ko_20180703.csv          0.009604           0.019208         0.024610\n",
       "wikilove_es_20180625.csv          0.018370           0.008183         0.025718\n",
       "wikilove_en_20180713.csv          0.013531           0.017264         0.027140\n",
       "wikithank_ar_20180623.csv         0.023409           0.004375         0.027191\n",
       "wikithank_tr_20180628.csv         0.026359           0.003040         0.028326\n",
       "wikithank_ro_20180704.csv         0.029215           0.000651         0.029773\n",
       "wikilove_vi_20180703.csv          0.011256           0.020142         0.031398\n",
       "wikithank_bn_20180702.csv         0.030090           0.002760         0.032850\n",
       "wikithank_pl_20180704.csv         0.038638           0.001087         0.039515\n",
       "wikithank_id_20180703.csv         0.043183           0.006513         0.047912\n",
       "wikithank_vi_20180703.csv         0.047970           0.003739         0.051282\n",
       "wikilove_tr_20180628.csv          0.036665           0.022064         0.056132\n",
       "wikilove_pt_20180627.csv          0.059546           0.037672         0.070213\n",
       "wikilove_sv_20180627.csv          0.133505           0.003093         0.135567"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nan_frac_df.sort_values('either_nan_frac')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sender_nan_frac      0.017438\n",
       "receiver_nan_frac    0.006289\n",
       "either_nan_frac      0.022120\n",
       "dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nan_frac_df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filled = df.fillna(nat_filler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'either_nan_frac': 0.0, 'receiver_nan_frac': 0.0, 'sender_nan_frac': 0.0}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frac_nan(df_filled)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}