mnist_vae.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VAE with MNIST Dataset\n",
    "\n",
    "- model=VAE() in vae.py\n",
    "- loss_function() in loss_function.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.utils.data\n",
    "from torch import optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import datasets, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "\n",
    "from vae import VAE\n",
    "from loss_function import loss_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim = 784\n",
    "hdims = 400\n",
    "seed = 25\n",
    "batch_size = 128\n",
    "epochs = 30\n",
    "zdims = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x1163734b0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load MNIST dataset\n",
    "\n",
    "- Dataset : numpy, pandas, rawdata?\n",
    "- DataLoader : batch단위로 데이터를 불러오는데 -> 편하게 해주는? Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('data', train=False, transform=transforms.ToTensor()),\n",
    "    batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = VAE(input_dim, zdims, hdims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch) : \n",
    "    \n",
    "    train_loss = 0\n",
    "    loss_train = []\n",
    "    loss_valid = []\n",
    "    \n",
    "    # get loss for whole validation set\n",
    "    loss_valid.append(validate(1))\n",
    "    \n",
    "    model.train() # toggle train mode\n",
    "    \n",
    "    # get batch loss for train set and backpropate\n",
    "    for batch_idx, (data, _) in enumerate(train_loader) :\n",
    "        data = Variable(data)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Propagate\n",
    "        recon_batch, mu, logvar = model(data)\n",
    "        \n",
    "        # Get loss value\n",
    "        loss = loss_function(recon_batch, data, mu, logvar, batch_size, input_dim)\n",
    "        \n",
    "        # Backpropagate\n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        # Update weights\n",
    "        optimizer.step()\n",
    "            \n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(\n",
    "          epoch, train_loss / len(train_loader.dataset)))\n",
    "      \n",
    "    # append to list 'loss_train' instance to plot later\n",
    "    loss_train.append(train_loss / len(train_loader.dataset))\n",
    "    \n",
    "    return loss_train, loss_valid\n",
    "    \n",
    "    \n",
    "with torch.no_grad() :\n",
    "    dfsdfsdkjfsldkjflsdkjflksf\n",
    "    \n",
    "def validate(epoch) : \n",
    "    model.eval() # toggle inference mode\n",
    "    valid_loss = 0\n",
    "    \n",
    "    \n",
    "    for i, (data, _) in enumerate(test_loader) :\n",
    "        data = Variable(data)\n",
    "        \n",
    "        # Propagate\n",
    "        recon_batch, mu, logvar = model(data)\n",
    "        \n",
    "        # Get loss value\n",
    "        valid_loss += loss_function(recon_batch, data, mu, logvar, batch_size, input_dim).item()\n",
    "\n",
    "    valid_loss /= len(test_loader.dataset)\n",
    "    print('====> Validation set loss: {:.4f}'.format(valid_loss))\n",
    "    \n",
    "    return valid_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====> Validation set loss: 0.0055\n",
      "====> Epoch: 1 Average loss: 0.0019\n",
      "====> Validation set loss: 0.0017\n",
      "====> Epoch: 2 Average loss: 0.0017\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 3 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 4 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 5 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 6 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 7 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 8 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 9 Average loss: 0.0016\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 10 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 11 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 12 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0016\n",
      "====> Epoch: 13 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 14 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 15 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 16 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 17 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 18 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 19 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 20 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 21 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 22 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 23 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 24 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 25 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 26 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 27 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 28 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 29 Average loss: 0.0015\n",
      "====> Validation set loss: 0.0015\n",
      "====> Epoch: 30 Average loss: 0.0015\n"
     ]
    }
   ],
   "source": [
    "loss_train = []\n",
    "loss_valid = []\n",
    "lowest_valid_loss = 0\n",
    "best_model = copy.deepcopy(model)\n",
    "\n",
    "for epoch in range(1, epochs+1) :\n",
    "    temp_train, temp_valid = train(epoch)\n",
    "    \n",
    "    if epoch == 1 :\n",
    "        lowest_valid_loss = temp_valid\n",
    "    else :\n",
    "        if temp_valid <= lowest_valid_loss :\n",
    "            lowest_valid_loss = temp_valid\n",
    "            best_model = copy.deepcopy(model)\n",
    "            \n",
    "    loss_train.append(temp_train)\n",
    "    loss_valid.append(temp_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check how model trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ8AAAEWCAYAAAC5XZqEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcXGWd7/HPr5au6j2dQJAskrAoWQgJhEUjEMTxxSIgKgEmqKDCyHUGHWcc0TuO6JU76HCRwUEQRhQRQQhG2RRRE1BHEYIxhARkSyAJSUgn3Unv1VW/+8c53anuVPWWrqpevu/Xq15nP/WcKuhvnueceh5zd0RERIopUuoCiIjI+KPwERGRolP4iIhI0Sl8RESk6BQ+IiJSdAofEREpOoWPiAw7M/u+mX2t1OWQkUvhI2OemW0ws/eUuhwispfCR0REik7hI+OamV1mZi+Z2U4ze8DMpoTrzcy+aWbbzWy3mT1rZnPDbWea2Toz22Nmm83sn/Oc+2oz+2HW8gwzczOLhcsrzezfzexP4Xv8zMwmZu1/jpk9Z2YN4b6zsrZtMLN/NrM1ZtZoZj82s2Qf1/kxM1tvZrvM7FEzOyRrm5vZlWb2ipntMLP/MLNIuC1iZv9qZhvDz+IHZlabdey7zOx/wjK+bmaXZL1tnZk9HH5OT5rZYf19tjJ+KHxk3DKzdwP/DiwBDgY2AveEm98LnAy8DagN96kPt30X+Dt3rwbmAr/Zj2J8BPhY+P6dwI1h2d4G3A18BjgQeAR40MzKso5dApwOzATmAZfkuc5zgS8CHwjP9dvw3NnOAxYCxwDnhmUiPOclwKnAoUAV8F/heQ8Bfg58KzzvfGB11jkvBL4C1AEvAdeE6/v6bGWcUPjIeLYUuN3dn3H3duALwDvMbAaQAqqBIwFz9/Xu/kZ4XAqYbWY17r7L3Z/ZjzLc6e5r3b0Z+BKwxMyiwAXAw+7+mLungOuAcuCdWcfe6O5b3H0n8CDBH/9cPgn8e3gNncD/BeZn136Ar7v7Tnd/DbgBuChcvxS43t1fcfcmgs/owrD29rfAr9z9bndPuXu9u2eHz3J3/1P4nndlla+vz1bGCYWPjGdTCGo7AIR/XOuBqe7+G4J/4d8EbDezW82sJtz1g8CZwEYze9zM3rEfZXg9a34jEAcOyFG2TLjv1Kz9t2bNtxDUSnI5BPjPsGmsAdgJWK9z9S7HlHC+RznC+RhwEDAdeLmPa8tZvn4+WxknFD4ynm0h+MMMgJlVApOAzQDufqO7HwvMJmgi+ly4/il3PxeYDPwUuDfP+ZuBiqzlt+TYZ3rW/FsJagU7cpTNwn03D/zyur1O0Ew4IetV7u7/00c5toTzPcoRbusEtoXnPWwI5cn72cr4ofCR8SJuZsmsV4zgvselZjbfzBIEzVFPuvsGMzvOzE4wszhBiLQBGTMrM7OlZlYbNoftBjJ53nM1cLKZvTW8Sf+FHPtcbGazzawC+CqwzN3TBIF2lpmdFpbhn4B24H9ynKM/twBfMLM5AGZWa2bn99rnc2ZWZ2bTgU8DPw7X3w38o5nNNLOq8DP6cVZT2nvMbImZxcxskpnla/rrlu+zHcJ1ySim8JHx4hGgNet1tbv/iuA+y/3AGwT/ir8w3L8GuA3YRdDUVA/8R7jtw8AGM9tNcD9laa43dPfHCP6IrwFWAQ/l2O1O4PsETVRJ4Mrw2BeAiwlu5u8AzgbOdveOwV64uy8Hvg7cE5Z5LXBGr91+FpZxNfAwwUMVALeHZXwCeJUgKP4hPO9rBM2P/0TQlLcaOHoARerrs5VxwjSYnEhpmNlK4Ifu/t8lLocDR7j7S6Ush4wvqvmIiEjRKXxERKTo1OwmIiJFp5qPiIgUXazUBRipDjjgAJ8xY0apiyEiMqqsWrVqh7sf2N9+Cp88ZsyYwdNPP13qYoiIjCpmtrH/vdTsJiIiJaDwERGRolP4iIhI0emej4iMGKlUik2bNtHW1lbqokg/kskk06ZNIx6PD+l4hY+IjBibNm2iurqaGTNmEHTkLSORu1NfX8+mTZuYOXPmkM6hZjcRGTHa2tqYNGmSgmeEMzMmTZq0XzVUhY+IjCgKntFhf78nhc9we3YZPHlrqUshIjKiKXyG2/oH4anbSl0KERmChoYGvv3tbw/p2DPPPJOGhoYB73/11Vdz3XXXDem9xgKFz3BL1kDb7lKXQkSGoK/w6ezs7PPYRx55hAkTJhSiWGOSwme4JWqgXeEjMhpdddVVvPzyy8yfP5/Pfe5zrFy5kpNOOolzzjmH2bNnA/D+97+fY489ljlz5nDrrXub2GfMmMGOHTvYsGEDs2bN4rLLLmPOnDm8973vpbW1tc/3Xb16NSeeeCLz5s3jvPPOY9euXQDceOONzJ49m3nz5nHhhcEgu48//jjz589n/vz5LFiwgD179hTo0ygsPWo93JK1kGqBdAqiQ3v+XUTgKw8+x7otw/sPudlTavjy2XPybr/22mtZu3Ytq1evBmDlypU888wzrF27tvuR4ttvv52JEyfS2trKcccdxwc/+EEmTZrU4zwvvvgid999N7fddhtLlizh/vvv5+KLL877vh/5yEf41re+xSmnnMK//du/8ZWvfIUbbriBa6+9lldffZVEItHdpHfddddx0003sWjRIpqamkgmk/v7sZSEaj7DLVkbTNX0JjImHH/88T1+y3LjjTdy9NFHc+KJJ/L666/z4osv7nPMzJkzmT9/PgDHHnssGzZsyHv+xsZGGhoaOOWUUwD46Ec/yhNPPAHAvHnzWLp0KT/84Q+JxYK6wqJFi/jsZz/LjTfeSENDQ/f60WZ0lnokS9QE0/ZGqJzU974ikldfNZRiqqys7J5fuXIlv/rVr/jDH/5ARUUFixcvzvlbl0Qi0T0fjUb7bXbL5+GHH+aJJ57gwQcf5JprruHZZ5/lqquu4qyzzuKRRx5h0aJFPProoxx55JFDOn8pqeYz3JJh+KjmIzLqVFdX93kPpbGxkbq6OioqKnj++ef54x//uN/vWVtbS11dHb/97W8BuPPOOznllFPIZDK8/vrrnHrqqXz961+nsbGRpqYmXn75ZY466ig+//nPc9xxx/H888/vdxlKQTWf4dZV82lrLG05RGTQJk2axKJFi5g7dy5nnHEGZ511Vo/tp59+OrfccguzZs3i7W9/OyeeeOKwvO8dd9zBJz/5SVpaWjj00EP53ve+Rzqd5uKLL6axsRF358orr2TChAl86UtfYsWKFUQiEebMmcMZZ5wxLGUoNnP3UpdhRFq4cKEPaTC5N9bAd06CC34Is84e/oKJjGHr169n1qxZpS6GDFCu78vMVrn7wv6OVbPbcFOzm4hIvxQ+w637gQOFj4hIPgqf4ZZQzUdEpD8Kn+EWjUFZlR44EBHpg8KnEBI1we98REQkJ4VPL2Z2tpnd2ti4H+GhzkVFRPqk8OnF3R9098tra2uHfhJ1LioyblRVVQGwZcsWPvShD+XcZ/HixfT3040bbriBlpaW7uXBDtGQz0gdukHhUwjJWt3zERlnpkyZwrJly4Z8fO/wGetDNCh8CkHNbiKj0lVXXcVNN93UvdxVa2hqauK0007jmGOO4aijjuJnP/vZPsdu2LCBuXPnAtDa2sqFF17IrFmzOO+883r07XbFFVewcOFC5syZw5e//GUg6Kx0y5YtnHrqqZx66qnA3iEaAK6//nrmzp3L3LlzueGGG7rfbzQP3aDudQpBzW4i++/nV8HWZ4f3nG85Cs64Nu/mCy64gM985jN86lOfAuDee+/l0UcfJZlMsnz5cmpqatixYwcnnngi55xzDmaW8zw333wzFRUVrF+/njVr1nDMMcd0b7vmmmuYOHEi6XSa0047jTVr1nDllVdy/fXXs2LFCg444IAe51q1ahXf+973ePLJJ3F3TjjhBE455RTq6upG9dANqvkUgmo+IqPSggUL2L59O1u2bOEvf/kLdXV1TJ8+HXfni1/8IvPmzeM973kPmzdvZtu2bXnP88QTT3SHwLx585g3b173tnvvvZdjjjmGBQsW8Nxzz7Fu3bo+y/S73/2O8847j8rKSqqqqvjABz7Q3QnpaB66QTWfQkjUQLodUm0QH50DPYmUXB81lEI6//zzWbZsGVu3buWCCy4A4K677uLNN99k1apVxONxZsyYkXMohf68+uqrXHfddTz11FPU1dVxySWXDOk8XUbz0A2q+RRC14ByanoTGXUuuOAC7rnnHpYtW8b5558PBLWGyZMnE4/HWbFiBRs3buzzHCeffDI/+tGPAFi7di1r1qwBYPfu3VRWVlJbW8u2bdv4+c9/3n1MvuEcTjrpJH7605/S0tJCc3Mzy5cv56STThr0dY20oRtU8ymE7NFMqyaXtiwiMihz5sxhz549TJ06lYMPPhiApUuXcvbZZ3PUUUexcOHCfmsAV1xxBZdeeimzZs1i1qxZHHvssQAcffTRLFiwgCOPPJLp06ezaNGi7mMuv/xyTj/9dKZMmcKKFSu61x9zzDFccsklHH/88QB84hOfYMGCBX02seUzkoZu0JAKeQx5SAWAF34Bd18Al/0Gph47vAUTGcM0pMLooiEVRpqkBpQTEemLwqcQspvdRERkHwqfQtCYPiJDplsBo8P+fk8Kn0LQaKYiQ5JMJqmvr1cAjXDuTn19/X798FRPuxVCWTVgqvmIDNK0adPYtGkTb775ZqmLIv1IJpNMmzZtyMcrfAohEgma3vTAgcigxONxZs6cWepiSBGo2a1Q1MWOiEheCp9CUeeiIiJ5KXwKJalmNxGRfBQ+haIB5URE8lL4FIqa3URE8lL4FIoeOBARyUvhUyhdNR/9WE5EZB8Kn0JJ1kKmE1ItpS6JiMiIo/ApFHWxIyKSl8KnUNS5qIhIXgqfQtGwCiIieSl8CiWhAeVERPJR+BRKV82nXeEjItKbwqdQ9MCBiEhe4yp8zGyWmd1iZsvM7IqCvpkeOBARyavg4WNmUTP7s5k9tB/nuN3MtpvZ2hzbTjezF8zsJTO7qq/zuPt6d/8ksARYNNTyDEhZJVhUNR8RkRyKUfP5NLA+1wYzm2xm1b3WHZ5j1+8Dp+c4PgrcBJwBzAYuMrPZZnaUmT3U6zU5POYc4GHgkf25qH6ZqWdrEZE8Cho+ZjYNOAv47zy7nAL81MwS4f6XAd/qvZO7PwHszHH88cBL7v6Ku3cA9wDnuvuz7v6+Xq/t4bkecPczgKV5yny2md3a2DgMoaHORUVEcip0zecG4F+ATK6N7n4f8CjwYzNbCnwMOH8Q558KvJ61vClcl5OZLTazG83sO+Sp+bj7g+5+eW1t7SCKkYc6FxURySlWqBOb2fuA7e6+yswW59vP3b9hZvcANwOHuXtTocrk7iuBlYU6/z4Star5iIjkUMiazyLgHDPbQNAc9m4z+2HvnczsJGAusBz48iDfYzMwPWt5WrhuZNCAciIiORUsfNz9C+4+zd1nABcCv3H3i7P3MbMFwK3AucClwCQz+9og3uYp4Agzm2lmZeH7PDAsFzAc1OwmIpJTqX/nUwEscfeX3T0DfATY2HsnM7sb+APwdjPbZGYfB3D3TuDvCe4brQfudffnilb6/iRq1MOBiEgOBbvnky3fvRZ3/32v5RRwW479Lurj3I9Q6MemhypZA+17IJOBSKlzXkRk5NBfxEJK1oJnoKNgz1CIiIxKCp9CUhc7IiI5KXwKSZ2LiojkpPApJNV8RERyUvgUUvdopnriTUQkm8KnkDSUtohITgqfQupudlPNR0Qkm8KnkPTAgYhITgqfQoolIRLXAwciIr0ofArJTJ2LiojkoPApNHUuKiKyD4VPoWk0UxGRfSh8Ck01HxGRfSh8Ck33fERE9qHwKTQNpS0isg+FT6Gp2U1EZB8Kn0JL1EDHHsikS10SEZERQ+FTaF39u6npTUSkm8Kn0NTFjojIPhQ+haYxfURE9qHwKTTVfERE9qHwKTQNKCcisg+FT6Gp2U1EZB8Kn0LTaKYiIvtQ+BSaRjMVEdmHwqfQYmXBoHKq+YiIdFP4FIM6FxUR6UHhUwwa00dEpIdBhY+Z1ZnZvEIVZsxS56IiIj30Gz5mttLMasxsIvAMcJuZXV/4oo0hqvmIiPQwkJpPrbvvBj4A/MDdTwDeU9hijTG65yMi0sNAwidmZgcDS4CHClyesUnNbiIiPQwkfL4KPAq85O5PmdmhwIuFLdYYo2Y3EZEeYv3t4O73AfdlLb8CfLCQhRpzkrWQaoF0CqLxUpdGRKTkBvLAwTfCBw7iZvZrM3vTzC4uRuGGm5nNMrNbzGyZmV1RtDdWFzsiIj0MpNntveEDB+8DNgCHA5/r7yAzS5rZn8zsL2b2nJl9ZaiFNLPbzWy7ma3Nse10M3vBzF4ys6v6Oo+7r3f3TxLcv1o01PIMmrrYERHpYUAPHITTs4D73H2gf0HbgXe7+9HAfOB0Mzsxewczm2xm1b3WHZ7jXN8HTu+90syiwE3AGcBs4CIzm21mR5nZQ71ek8NjzgEeBh4Z4HXsP43pIyLSw0DC5yEzex44Fvi1mR0ItPV3kAeawsV4+PJeu50C/NTMEgBmdhnwrRznegLYmeNtjid4EOIVd+8A7gHOdfdn3f19vV7bw3M94O5nAEtzldvMzjazWxsbh7GWomEVRER66Dd83P0q4J3AQndPAc3AuQM5uZlFzWw1sB14zN2f7HXu+wiepPuxmS0FPgacP4jyTwVez1reFK7LV57FZnajmX2HPDUfd3/Q3S+vra0dRDH6oQHlRER66PdpNzOLAxcDJ5sZwOPALQM5ubungflmNgFYbmZz3X1tr32+YWb3ADcDh2XVloadu68EVhbq/Hmp2U1EpIeBNLvdTNDk9u3wdUy4bsDcvQFYQe77NicBc4HlwJcHc15gMzA9a3lauG5kUbObiEgPAwmf49z9o+7+m/B1KXBcfweZ2YFhjQczKwf+Bni+1z4LgFsJmvEuBSaZ2dcGUf6ngCPMbKaZlQEXAg8M4vjiSKjmIyKSbSDhkzazw7oWwh4O0gM47mBghZmtIQiJx9y9d/c8FcASd3/Z3TPAR4CNvU9kZncDfwDebmabzOzjAO7eCfw9wX2j9cC97v7cAMpWXNEYxCtV8xERCfV7z4fgNz0rzOwVwIBDCGopfXL3NcCCfvb5fa/lFHBbjv0u6uMcj1DMx6aHKlkLbQ2lLoWIyIgwkO51fm1mRwBvD1e94O7thS3WGKTORUVEuuUNHzP7QJ5Nh5sZ7v6TApVpbFLnoiIi3fqq+ZzdxzYHFD6DkayBlly/kxURGX/yhk/4VJsMl2Qt7Hy11KUQERkRBvK0mwwHNbuJiHRT+BSLHjgQEemm8CmWRA2k2yHVb5+sIiJj3kD6dsv11Fsj8GxXT9EyAF2di7bvhniytGURESmxgfzI9OPAOwj6ZgNYDKwCZprZV939zgKVbWzJHs20anJpyyIiUmIDCZ8YMMvdtwGY2UHAD4ATgCcAhc9AaDRTEZFuA7nnM70reELbw3U7gVRhijUGaVgFEZFuA6n5rDSzh4D7wuUPhesqAXVWNlAaUE5EpNtAwudTwAeAd4XLdwD3u7sDpxaqYGOOxvQREek2kI5F3cx+B3QQdKvzpzB4ZDDU7CYi0q3fez5mtgT4E0Fz2xLgSTP7UKELNuaUVQOmmo+ICANrdvvfBKOZbodghFLgV8CyQhZszIlEIFGtmo+ICAN72i3S68ek9QM8TnpL1uqBAxERBlbz+YWZPQrcHS5fwGgYOXQkUueiIiLAwB44+JyZfRBYFK661d2XF7ZYY1SyRjUfEREGVvPB3e8H7i9wWca+RA00bS11KURESq6vYbT3EDxavc8mgiewawpWqrEqWQs7Xih1KURESq6vkUyri1mQcUFj+oiIAHpqrbi6HjjQb3RFZJxT+BRTsgYynZBqLXVJRERKSuFTTOpcVEQEUPgUlzoXFREBFD7FlT2aqYjIOKbwKSaNZioiAih8ikv3fEREAIVPcWlMHxERQOFTXHrgQEQEUPgUV1klWFQ1HxEZ9xQ+xWQWNL2p5iMi45zCp9gSGlZBREThU2zqXFREROFTdIlaNbuJyLin8Ck21XxERBQ+RZes1T0fERn3FD7FlqhR9zoiMu4pfIotWQPtezSgnIiMawqfYkvUgGego6nUJRERKRmFT7Gpc1EREYVP0alzURERhU/RqXNRERGFT9FpNFMREYVP0XWFj2o+IjKOKXyKravZra2htOUQESkhhU+x6YEDERGFT9HFkhCJq9lNRMY1hU+xmYX9uyl8RGT8UviUQlIDyonI+DauwsfMZpnZLWa2zMyuKFlBEhpKW0TGt4KFj5lNN7MVZrbOzJ4zs0/vx7luN7PtZrY2x7bTzewFM3vJzK7q6zzuvt7dPwksARYNtTz7TWP6iMg4V8iaTyfwT+4+GzgR+JSZzc7ewcwmm1l1r3WH5zjX94HTe680syhwE3AGMBu4yMxmm9lRZvZQr9fk8JhzgIeBR/b/EodINR8RGecKFj7u/oa7PxPO7wHWA1N77XYK8FMzSwCY2WXAt3Kc6wlgZ463OR54yd1fcfcO4B7gXHd/1t3f1+u1PTzXA+5+BrA0V7nN7Gwzu7WxsYD3ZJITdM9HRMa1otzzMbMZwALgyez17n4f8CjwYzNbCnwMOH8Qp54KvJ61vIl9Ay67HIvN7EYz+w55aj7u/qC7X15bWzuIYgySmt1EZJyLFfoNzKwKuB/4jLvv8xfX3b9hZvcANwOHuXvBBrpx95XAykKdf8ASNdCxBzJpiERLXRoRkaIraM3HzOIEwXOXu/8kzz4nAXOB5cCXB/kWm4HpWcvTwnUjW1cvB+17SlsOEZESKeTTbgZ8F1jv7tfn2WcBcCtwLnApMMnMvjaIt3kKOMLMZppZGXAh8MD+lbwINKCciIxzhaz5LAI+DLzbzFaHrzN77VMBLHH3l909A3wE2Nj7RGZ2N/AH4O1mtsnMPg7g7p3A3xPcN1oP3OvuzxXukoaJxvQRkXGuYPd83P13gPWzz+97LaeA23Lsd1Ef53iEUj42PRTqXFRExrlx1cNBsXSmM33voJqPiIxzCp9hdvUDz3HFXc/0vZNGMxWRcU7hM8zeUpvksXXb+P1LO/LvpAcORGScU/gMs0veOYNpdeX8n4fWkc547p26m90UPiIyPil8hlkyHuWqM47k+a17uO/p13PvFCsLBpVTs5uIjFMKnwI466iDOfaQOq775V9pau/MvVOyVg8ciMi4pfApADPjX8+axY6mdm5Z+XLunRIaUE5Exi+FT4EseGsd586fwm2/fYXNDa377qDORUVkHFP4FNC/nH4kAN/4xfP7btSYPiIyjil8CmjqhHIuO+lQfrZ6C39+bVfPjar5iMg4pvApsE8uPowDqxN87eH1uGc9ep2s1T0fERm3FD4FVpWI8c/vfRurNu7i4Wff2LtBzW4iMo4pfIrgQ8dO58i3VHPtz5+nLZUOViZrIdUC6VRpCyciUgIKnyKIRowvvW82m3a18r3fbwhWJjSgnIiMXwqfIll0+AG8Z9ZkblrxEjua2rP6d2sobcFEREpA4VNEXzhzFm2pNN987K8a00dExjWFTxEddmAVF594CHf/6TVeaw7H8dNDByIyDil8iuzTpx1BVSLGfz3dHKz4+efh2WWQSZe2YCIiRaTwKbK6yjKuPO0I7n01wbp3fjMInfs/Dv+1EFbdAZ3tpS6iiEjBKXxK4CPvmMGMSRV8eu1htF3+e1hyZ/D024NXwn/Ohz98GzqaS11MEZGCUfiUQFkswhfPnMWL25s47prf8A9/mc6DJ/yIlgvug4mHwqNfgG/Ohcf/A1p39X9CEZFRxnp0+SLdFi5c6E8//XRB3+O3L77Jw2ve4LF126hv7iAeNd552AH87ZQ3WLz9ThKvPAZl1XDcx+HYS6BuBpgVtEwiIvvDzFa5+8J+91P45FaM8OmSzjh/fm0Xv1y3jUef28rG+hYAzjt4J/8r/iCHv/kY5hlI1MLkWXDQbJg8Gw6aE0zLJxSlnCIi/VH47Kdihk82d+fF7U388rmt/HLdNtZsauQQ28pZlS+wMLmFI+w1Dmp9hbLOrJ4RaqaGYTQbJs+BSYcFzXfldaopiUhRKXz2U6nCp7ctDa08tm4bz7y2ixe27uHlN5tIpTMczE5mRV/jhMptzC/bzKGZjUxq3UjEs/qKS9TCxBlQNxMmzgwCqWu+egpEdMtPRIaXwmc/jZTw6S2VzrCxvpkXtjbxwrY9/HXrHv66fQ8bdjQT8U4OsW3MjGzlqPJdHFn2JjOi2zmo8w1q2rYQ8c69J4omYMJ0qJkS1Jxqpuydrz44mFZMUkCJyKAMNHxixSiMDJ94NMLhk6s5fHI1Z3Fw9/q2VJqX32zixW1NbKhvZkN9Cyvrm3mtvoX65g6ipDnY6jnEtjEnuZM5iR0c0lHPgTt2MuGNF6lo3455rx+6Rsv2BlHNwT2DqjoMq6qDIKr/jERkcPRXY4xIxqPMmVLLnCm1+2zb3ZbitfoWNta3sKG+mY31zdxV38LmXa1s3d1GOuNEyDCJRg62nRyebORt5XuYEW9giu3iwN07qKl/ivLWrUQyHT1PbpEggGqmBEFVdRBUTAzuN+V6JSdArKxIn4qIjFQKn3GgJhln7tRa5k7dN5g60xm27WlnS0MrWxpa2dzQyuZdrTzZ0MryhjY272ylqb2ruc6pYw9vsV0cnmzkiORuDilrZGpkFwc21zOhYR3lqd8RT+0Ons7Lp6wqCKKKiVBxQNC8VxlOu+cP2LsuUQ2RmB6eEBlDFD7jXCwaYeqEcqZOKM+7T2Nrim2729ja2MbW3W1sC6drdrfxy91tbNvdzo6mdrpuHxoZqmml1pp4S7yV6eXtTClr4y1lLRwYbWVipJkJtofqzG4qGnaQ3P5XYm07iaT66dUhEodo+IrEg2bBaKznfDQBsfAVTQS1rFgy2B5L7J1PVAXBVj5xb+hVTAyW1YwoUnD6v0z6VVsep7Y8ztsOqs67Tyqd4c097by5p5365nZ2NHVQ39TBjqZ26pva+UtzB79u6qB+dzv1zR2kM/s+6JKgg2mJVmaUtzI92cKUeDNCYCMsAAAMqUlEQVQHRZuYEG2nPOokI2mSkQzJaIaEdVJmGcosTZxOYqSJZFKQbofOjmCoiq75dHvQZ15nO6Q7oLMNMp05riKUrM0KpolB10eJ6n1fZVXhfLg9WRPU6OL5g1xEAgofGRbxaIQpE8qZ0kcNqksm4zS2pqhv7mBncwc7m4NA2tnU0b3upeYOnmzuYGdjOw0tKdo7+2jGC1WURbuDsrY8zoTKcFpRRm15nJryOBO6tsfT1LKbmsweKjobSHQ0YK07oWUntNQHr9ad0LQN6l8KRpxtb4LO1v4/jFgyuLfV437XhL3T5ITgXtlAdNXSEtVBbxeJqr2hV1al+2cyail8pOgiEaOusoy6yoH/4ezozLCnLcWets7wlWJ3OO1at7stxe7WFA2tKRpbU2zY0UJja4qG1g7aUn2HV8SSVCXeSnXyMKoSMaqSsWBaFaNqUozKcF11PENdNEVNpJXaSCtV1kYlrVR6M+WZZhKp3cQ6GrHWXUG/fK0N0LAR3lgdLKda9vfj6ylaFgZRZTAfiYdNkbG9zZSRWFZTZfi/fCYd1P7SqWCaSUOma75rOR02V2Y3XSZzLIfNmfEkxMqDabwiXFceTiuytpeHy+G2gTzOn8lAx569n2lbw975jubgM+i6j9gd+BOD95QRSeEjo0JZLMKkqgSTqhJDOr4tlWZ3GEoNranu0Gpq76QpnPZebmhN8fquFprbO2luT2c9eNFbHJgQvqZiBhXxKBWJGJVlUSrKYlSUR6mYEKM2nmZitJ3yOJTHYyTiEcrjUcrLoiRjUcrLIiRjUZJlUcrjEcotTSWtJDItlHsrZekWIqmmvTWxjq5pc9CkmElBuitAwvnOtmD/rmUziETDgApDKhIN75OFYRWJBrWzruPTHdDRFNQIO9v3Nml2toVNmm3Q+1H9gcoOrHhWOKVTe0OmrRH6eoilr3Nnh1JZ1d6w7J72CtFYMvgcusI5nQo/uzzzkVgQ/mVV4TTXfFVwXZEoYMFna9ZzPns5EgvKE4kO7TMdBRQ+Mi4k41GS8SiTa4b+L+FMxmlJpbvDqam9k+as0GrpCEKqtaOT5o5093JLRyctHWkaW1O80dBJS0eG1lSalo7Wfmtk+4qTiE2iomwyFWUxkvEIFWUxysuilMejVITT8vJod6h1beteDj+L7MBLxMLlcL9oZAhPFqZTQc0u1RY0T6bCV2fb3vWp1qxtLVnTtl7rmoM/2BMP3dtUmd1smT1fVhkG486wNtT1ylpuCZd3b94blr2nDOQH9xY+3JJVo8ykw/AvwFhcFt33AZquh2qiZcELD4J5n1ev9RYNH9gp2/sPja5zdL1i4fRd/wi104b/erIofEQGKBKxoCkuMXz/22QyTltnmtaONC0dadpSwbQ1tXddMB8EWPY+wbbO7v0aWjrY0rD32NZU8BpKJyZl0QiJeIRkPEoiFgnDO6yVZa3L3qcsFoRYIhYJX+WUxSpJxKLhtghl5Xv36V7XdVw8ErxvLIIN9rH68gn798fSPQjPrjBKt/dqtuwKnD5qIulUEELdr6ae86mWfYOh6709Q48QyXTmeFgm+wGacJruCGtO+V5hTQoLzpvuCF8p6GiBdENYi+s6Xzh/3CeG/lkOkMJHpIQiEQua5cpiTCrA+d2d9s5Mdxh1hVdbGExtqUw4TdOeY11bKkN7Kk1bZzDfddyulo692zvTtKcytKczdAzgwZCBKItFSEQjxGNBIMVjFkyjewMrHg2mQVBmh14YiuE0kR2K8Z7Hdc3Ho0YiFiEejVIWqyIerSEeiRCNGrGIEY9GiBj0GYnRePhgiXqZHwiFj8gYZmbdTY51RXg/d6cjnaG9MwiivdMwoMLljnSw3JHOhOvTtIf7t4f7pzqdjnSajs4MqbSHx2XC5WDa1N4ZrA+Pa0ul9x6fHv5+K+NRIxqxrGCK7F0XjRCLGLFosK5rvmtaFrUewVcW2/tKZC9HI5TFosSj1h2y2SHZFZrBOiNiwft3v8yIZE1jWdtiERt8rbJAFD4iMmzMLKxllP5GeToTBFZXILWl0qTCYEx1h1hXwAWhmQoDLpUOtqUzXVOnM52hM+N0ZpxUOkM646TSwfp0xkmF+6TSTmcmQ2fau9+vuSNNqut9w/fuDs1hrDEORFcIxaORMDSDEO2ej0a45eJjOHxy/t/1DQeFj4iMSdGIdT9wMdK5exiEPWt2XWGV6grHdO9tTiYThGPaw/lw2hmuz7iTzkA6E4Zn2kmF4dgdqNnrMhmS8cJ/ZgofEZESMzPKYkEzG0P7NcGoo8FaRESk6BQ+IiJSdAofEREpOoWPiIgUncJHRESKTuEjIiJFp/AREZGiU/iIiEjRmQ+ly9txwMzeBDYO8fADgB3DWJyRYKxd01i7Hhh716TrGflyXdMh7n5gfwcqfArAzJ5294WlLsdwGmvXNNauB8beNel6Rr79uSY1u4mISNEpfEREpOgUPoVxa6kLUABj7ZrG2vXA2LsmXc/IN+Rr0j0fEREpOtV8RESk6BQ+IiJSdAqfYWZmp5vZC2b2kpldVery7C8z22Bmz5rZajN7utTlGQozu93MtpvZ2qx1E83sMTN7MZzWlbKMg5Hneq42s83h97TazM4sZRkHw8ymm9kKM1tnZs+Z2afD9aP5O8p3TaPyezKzpJn9ycz+El7PV8L1M83syfDv3Y/NrGzA59Q9n+FjZlHgr8DfAJuAp4CL3H1dSQu2H8xsA7DQ3Uftj+PM7GSgCfiBu88N130D2Onu14b/SKhz98+XspwDled6rgaa3P26UpZtKMzsYOBgd3/GzKqBVcD7gUsYvd9Rvmtawij8nszMgEp3bzKzOPA74NPAZ4GfuPs9ZnYL8Bd3v3kg51TNZ3gdD7zk7q+4ewdwD3Buics07rn7E8DOXqvPBe4I5+8g+MMwKuS5nlHL3d9w92fC+T3AemAqo/s7yndNo5IHmsLFePhy4N3AsnD9oL4jhc/wmgq8nrW8iVH8H1zIgV+a2Sozu7zUhRlGB7n7G+H8VuCgUhZmmPy9ma0Jm+VGTRNVNjObASwAnmSMfEe9rglG6fdkZlEzWw1sBx4DXgYa3L0z3GVQf+8UPtKfd7n7McAZwKfCJp8xxYO259He/nwzcBgwH3gD+H+lLc7gmVkVcD/wGXffnb1ttH5HOa5p1H5P7p529/nANIJWniP353wKn+G1GZietTwtXDdqufvmcLodWE7wH91YsC1sl+9qn99e4vLsF3ffFv5xyAC3Mcq+p/A+wv3AXe7+k3D1qP6Ocl3TaP+eANy9AVgBvAOYYGaxcNOg/t4pfIbXU8AR4RMgZcCFwAMlLtOQmVlleLMUM6sE3gus7fuoUeMB4KPh/EeBn5WwLPut64906DxG0fcU3sz+LrDe3a/P2jRqv6N81zRavyczO9DMJoTz5QQPVa0nCKEPhbsN6jvS027DLHx08gYgCtzu7teUuEhDZmaHEtR2AGLAj0bj9ZjZ3cBigu7ftwFfBn4K3Au8lWDojCXuPipu4ue5nsUETTkObAD+Lut+yYhmZu8Cfgs8C2TC1V8kuEcyWr+jfNd0EaPwezKzeQQPFEQJKi33uvtXw78R9wATgT8DF7t7+4DOqfAREZFiU7ObiIgUncJHRESKTuEjIiJFp/AREZGiU/iIiEjRKXxExggzW2xmD5W6HCIDofAREZGiU/iIFJmZXRyOjbLazL4TdtjYZGbfDMdK+bWZHRjuO9/M/hh2RLm8qyNKMzvczH4Vjq/yjJkdFp6+ysyWmdnzZnZX+Et7zOzacGyZNWY2qrrzl7FJ4SNSRGY2C7gAWBR20pgGlgKVwNPuPgd4nKDXAoAfAJ9393kEv5bvWn8XcJO7Hw28k6CTSgh6T/4MMBs4FFhkZpMIunKZE57na4W9SpH+KXxEius04FjgqbB7+tMIQiID/Djc54fAu8ysFpjg7o+H6+8ATg7725vq7ssB3L3N3VvCff7k7pvCjitXAzOARqAN+K6ZfQDo2lekZBQ+IsVlwB3uPj98vd3dr86x31D7vcruVysNxMLxVo4nGPTrfcAvhnhukWGj8BEprl8DHzKzyQBmNtHMDiH4f7Grd+C/BX7n7o3ALjM7KVz/YeDxcGTMTWb2/vAcCTOryPeG4Zgyte7+CPCPwNGFuDCRwYj1v4uIDBd3X2dm/0owOmwESAGfApqB48Nt2wnuC0HQTf0tYbi8Alwarv8w8B0z+2p4jvP7eNtq4GdmliSoeX12mC9LZNDUq7XICGBmTe5eVepyiBSLmt1ERKToVPMREZGiU81HRESKTuEjIiJFp/AREZGiU/iIiEjRKXxERKTo/j/Z+vqRwQY/zwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_train, label='train loss')\n",
    "plt.plot(loss_valid, label='validation loss')\n",
    "plt.title('Loss upon epochs')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('log loss')\n",
    "plt.yscale('log')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## See how latent variables distribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}