Attentional RNN (To input) - Save for using in Java.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parameters = {}\n",
    "parameters['seq_length'] = 5\n",
    "parameters['n_input'] = 3\n",
    "parameters['n_output'] = 3\n",
    "parameters['n_hidden'] = 4\n",
    "parameters['init_stdev'] = 0.1\n",
    "parameters['learning_rate'] = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Some auxilar functions\n",
    "def _seq_length(sequence):\n",
    "    used = tf.sign(tf.reduce_max(tf.abs(sequence), reduction_indices=2))\n",
    "    length = tf.reduce_sum(used, reduction_indices=1)\n",
    "    length = tf.cast(length, tf.int32)\n",
    "    return length\n",
    "\n",
    "def _last_relevant(output, length):\n",
    "    batch_size = tf.shape(output)[0]\n",
    "    max_length = tf.shape(output)[1]\n",
    "    out_size = int(output.get_shape()[2])\n",
    "    index = tf.range(0, batch_size) * max_length + (length - 1)\n",
    "    flat = tf.reshape(output, [-1, out_size])\n",
    "    relevant = tf.gather(flat, index)\n",
    "\n",
    "    return relevant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-2f54572c4382>:38: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Define placeholders\n",
    "x = tf.placeholder(\"float\", [None, parameters['seq_length'], parameters['n_input']], name='x')\n",
    "y = tf.placeholder(\"float\", [None, parameters['n_output']], name='y')\n",
    "\n",
    "# Define weights and bias - For now we will try with attention to hidden state \n",
    "weights = {\n",
    "    'alphas': tf.Variable(tf.random_normal([parameters['n_hidden'], 1], stddev=parameters['init_stdev'])),\n",
    "    'out': tf.Variable(tf.random_normal([parameters['n_input'], parameters['n_output']], stddev=parameters['init_stdev']), name='w_out')\n",
    "        }\n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([parameters['n_output']]), name='b_out'),\n",
    "    'alphas': tf.Variable(tf.random_normal([1]), name='b_alphas')\n",
    "}\n",
    "\n",
    "# Define RNN\n",
    "rnn_cell = tf.contrib.rnn.LSTMCell(parameters['n_hidden'])\n",
    "outputs, states = tf.nn.dynamic_rnn(\n",
    "    rnn_cell,\n",
    "    x,\n",
    "    dtype=tf.float32,\n",
    "    sequence_length=_seq_length(x)\n",
    ")\n",
    "\n",
    "# Define attention weihts\n",
    "outputs_reshaped = tf.reshape(outputs, [-1, int(outputs.get_shape()[2])])\n",
    "ejs = tf.matmul(outputs_reshaped, weights['alphas']) + biases['alphas'] \n",
    "ejs_reshaped = tf.reshape(ejs, [-1, int(outputs.get_shape()[1])])\n",
    "alphas = tf.nn.softmax(ejs_reshaped, name='attention_weights') \n",
    "reshaped_alphas = tf.reshape(alphas, [-1, 1])\n",
    "# Define context\n",
    "x_reshaped = tf.reshape(x, [-1, int(x.get_shape()[2])])\n",
    "context = reshaped_alphas * x_reshaped\n",
    "context_reshaped = tf.reshape(context, [-1, parameters['seq_length'], int(context.get_shape()[1])])\n",
    "context_reduced = tf.reduce_sum(context_reshaped, axis= 1)\n",
    "# Define logits and loss\n",
    "logits = tf.matmul(context_reduced, weights['out']) + biases['out']\n",
    "pred_prob = tf.nn.softmax(logits, name=\"predictions\")\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "\n",
    "#Define optimizer\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=parameters['learning_rate']).minimize(loss)\n",
    "\n",
    "# Initialization\n",
    "init = tf.global_variables_initializer()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Sample 1 - The typical example, always the second element, which leads to the output of the second element to be one\n",
    "x1 = np.array([[0,1,0], [0,1,0],[0,1,0],[0,1,0],[0,1,0]])\n",
    "y1 = np.array([0,1,0])\n",
    "# Samples 2-6 - If in some point the first element is 1, then the output will be one for the first element\n",
    "x2 = np.array([[1,0,0], [0,1,0],[0,1,0],[0,1,0],[0,1,0]])\n",
    "y2 = np.array([1,0,0])\n",
    "x3 = np.array([[0,1,0], [1,0,0],[0,1,0],[0,1,0],[0,1,0]])\n",
    "y3 = np.array([1,0,0])\n",
    "x4 = np.array([[0,1,0], [0,1,0],[1,0,0],[0,1,0],[0,1,0]])\n",
    "y4 = np.array([1,0,0])\n",
    "x5 = np.array([[0,1,0], [0,1,0],[0,1,0],[1,0,0],[0,1,0]])\n",
    "y5 = np.array([1,0,0])\n",
    "x6 = np.array([[0,1,0], [0,1,0],[0,1,0],[0,1,0],[1,0,0]])\n",
    "y6 = np.array([1,0,0])\n",
    "# Samples 7 - 11 - If in some point the last element is 1, then the output will be one for the first element\n",
    "x7 = np.array([[0,0,1], [0,1,0],[0,1,0],[0,1,0],[0,1,0]])\n",
    "y7 = np.array([0,0,1])\n",
    "x8 = np.array([[0,1,0], [0,0,1],[0,1,0],[0,1,0],[0,1,0]])\n",
    "y8 = np.array([0,0,1])\n",
    "x9 = np.array([[0,1,0], [0,1,0],[0,0,1],[0,1,0],[0,1,0]])\n",
    "y9 = np.array([0,0,1])\n",
    "x10 = np.array([[0,1,0], [0,1,0],[0,1,0],[0,0,1],[0,1,0]])\n",
    "y10 = np.array([0,0,1])\n",
    "x11 = np.array([[0,1,0], [0,1,0],[0,1,0],[0,1,0],[0,0,1]])\n",
    "y11 = np.array([0,0,1])\n",
    "\n",
    "X = [x1, np.copy(x1), np.copy(x1), np.copy(x1), np.copy(x1), np.copy(x1), np.copy(x1), np.copy(x1), \n",
    "    x2, x3, x4, x5, x6,\n",
    "    x7, x8, x9, x10, x11]\n",
    "Y = [y1, np.copy(y1), np.copy(y1), np.copy(y1), np.copy(y1), np.copy(y1), np.copy(y1), np.copy(y1), \n",
    "    y2, y3, y4, y5, y6,\n",
    "    y7, y8, y9, y10, y11]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1, Loss= 1.2369\n",
      "Step 20, Loss= 1.0623\n",
      "Step 40, Loss= 0.9454\n",
      "Step 60, Loss= 0.6732\n",
      "Step 80, Loss= 0.4003\n",
      "Step 100, Loss= 0.2591\n",
      "Step 120, Loss= 0.1862\n",
      "Step 140, Loss= 0.1432\n",
      "Step 160, Loss= 0.1151\n",
      "Step 180, Loss= 0.0953\n",
      "Step 200, Loss= 0.0807\n",
      "Step 220, Loss= 0.0695\n",
      "Step 240, Loss= 0.0607\n",
      "Step 260, Loss= 0.0536\n",
      "Step 280, Loss= 0.0478\n",
      "Optimization Finished!\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: b'models/attentionRNN2/SavedModelBuilder/saved_model.pb'\n"
     ]
    }
   ],
   "source": [
    "# Start training\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    # Run the initializer\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(1, 300):\n",
    "        batch_x = np.array(X)\n",
    "        batch_y = np.array(Y)\n",
    "        # Run optimization op (backprop)\n",
    "        a = sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n",
    "        if step % 20 == 0 or step == 1:\n",
    "            # Calculate batch loss and accuracy\n",
    "            train_loss = sess.run(loss, feed_dict={x: batch_x, y: batch_y})\n",
    "            print(\"Step \" + str(step) + \", Loss= {:.4f}\".format(train_loss))\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "   \n",
    "    # Once trained - Get attention weights for the training samples\n",
    "    attention_weights = sess.run(alphas, feed_dict={x: batch_x})\n",
    "\n",
    "    # Saved Model Builder \n",
    "    export_path = \"models/attentionRNN2/SavedModelBuilder/\"\n",
    "    builder = tf.saved_model.builder.SavedModelBuilder(export_path)\n",
    "    builder.add_meta_graph_and_variables(\n",
    "          sess, [tf.saved_model.tag_constants.SERVING])\n",
    "    builder.save()\n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check attention weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  3.52991700e-01,   1.88761383e-01,   1.55129313e-01,\n",
       "          1.52590171e-01,   1.50527418e-01],\n",
       "       [  9.94235814e-01,   2.82458030e-03,   1.18486839e-03,\n",
       "          8.85288464e-04,   8.69476702e-04],\n",
       "       [  1.17540313e-03,   9.95346606e-01,   2.22401810e-03,\n",
       "          7.30713364e-04,   5.23235532e-04],\n",
       "       [  9.98247648e-04,   5.33810526e-04,   9.95851040e-01,\n",
       "          1.98952458e-03,   6.27387140e-04],\n",
       "       [  1.01222249e-03,   5.41283574e-04,   4.44841542e-04,\n",
       "          9.96020734e-01,   1.98098691e-03],\n",
       "       [  1.00754027e-03,   5.38779481e-04,   4.42783872e-04,\n",
       "          4.35536407e-04,   9.97575343e-01],\n",
       "       [  9.95554984e-01,   2.27934658e-03,   8.86101392e-04,\n",
       "          6.46104978e-04,   6.33479445e-04],\n",
       "       [  8.80833017e-04,   9.96336222e-01,   1.83612795e-03,\n",
       "          5.53057063e-04,   3.93821130e-04],\n",
       "       [  7.71604537e-04,   4.12613532e-04,   9.96635497e-01,\n",
       "          1.69179449e-03,   4.88511403e-04],\n",
       "       [  7.76475761e-04,   4.15218383e-04,   3.41237901e-04,\n",
       "          9.96790349e-01,   1.67665980e-03],\n",
       "       [  7.74430169e-04,   4.14124486e-04,   3.40338913e-04,\n",
       "          3.34768440e-04,   9.98136282e-01]], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "[ 0.3529917   0.18876138  0.15512931  0.15259017  0.15052742]\n"
     ]
    }
   ],
   "source": [
    "print(X[0])\n",
    "print(attention_weights[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "[ 0.3529917   0.18876138  0.15512931  0.15259017  0.15052742]\n"
     ]
    }
   ],
   "source": [
    "print(X[1])\n",
    "print(attention_weights[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "[  9.94235814e-01   2.82458030e-03   1.18486839e-03   8.85288464e-04\n",
      "   8.69476702e-04]\n"
     ]
    }
   ],
   "source": [
    "print(X[8])\n",
    "print(attention_weights[8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0]\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "[  1.17540313e-03   9.95346606e-01   2.22401810e-03   7.30713364e-04\n",
      "   5.23235532e-04]\n"
     ]
    }
   ],
   "source": [
    "print(X[9])\n",
    "print(attention_weights[9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0]\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "[  9.98247648e-04   5.33810526e-04   9.95851040e-01   1.98952458e-03\n",
      "   6.27387140e-04]\n"
     ]
    }
   ],
   "source": [
    "print(X[10])\n",
    "print(attention_weights[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 0 1]]\n",
      "[  7.74430169e-04   4.14124486e-04   3.40338913e-04   3.34768440e-04\n",
      "   9.98136282e-01]\n"
     ]
    }
   ],
   "source": [
    "print(X[-1])\n",
    "print(attention_weights[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}