END2_Session_4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "END2_Session_4.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JahnaviRamagiri/END2.0/blob/main/END2_Session_4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAM_ur4Rw8VK"
      },
      "source": [
        "!pip install -q spacy==3.0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBvIT0azx8YA"
      },
      "source": [
        "!python -m spacy download en_core_web_sm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SPhj6gnAnT2"
      },
      "source": [
        "import torch\n",
        "from torchtext.legacy import data\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "TEXT = data.Field(tokenize = 'spacy',\n",
        "                  tokenizer_language = 'en_core_web_sm')\n",
        "LABEL = data.LabelField(dtype = torch.float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edVAOwR6vpmA"
      },
      "source": [
        "\n",
        "def load_data(opt):\n",
        "    # use torchtext to load data, no need to download dataset\n",
        "    print(\"loading {} dataset\".format(opt.dataset))\n",
        "    # set up fields\n",
        "    text = data.Field(lower=True, include_lengths=True, batch_first=True, fix_length=opt.max_seq_len)\n",
        "    label = data.Field(sequential=False)\n",
        "\n",
        "    # make splits for data\n",
        "    train, test = datasets.IMDB.splits(text, label)\n",
        "\n",
        "    # build the vocabulary\n",
        "    text.build_vocab(train, vectors=GloVe(name='6B', dim=300))\n",
        "    label.build_vocab(train)\n",
        "\n",
        "    # print vocab information\n",
        "    print('len(TEXT.vocab)', len(text.vocab))\n",
        "    print('TEXT.vocab.vectors.size()', text.vocab.vectors.size())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwn4oStE6PzV"
      },
      "source": [
        "from torchtext.legacy import datasets\n",
        "\n",
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5DLJ86m56Xdn",
        "outputId": "7ca1b90c-ba44-48eb-fd13-e0365053cad5"
      },
      "source": [
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples: 25000\n",
            "Number of testing examples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iXTWwqXA6rP2",
        "outputId": "83f866bc-5ddf-4981-9551-d39f3829415b"
      },
      "source": [
        "print(vars(train_data.examples[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': ['Who', 'would', \"n't\", 'want', 'to', 'go', 'on', 'road', 'trip', 'with', 'Peter', 'Falk', '?', 'That', 'guy', \"'s\", 'right', 'eye', 'has', 'more', 'character', 'than', 'most', 'actors', 'today', '.', 'This', 'is', 'the', 'kind', 'of', 'funny', 'and', 'touching', 'movie', 'we', 'are', 'all', 'looking', 'for', 'as', 'a', 'counterbalance', 'to', 'all', 'the', 'bombastic', 'special', 'effects', 'bores', '.', 'Women', 'are', 'going', 'to', 'love', 'it', 'for', 'all', 'the', 'wake', '-', 'up', 'romance', 'advice', 'for', 'men', ',', 'and', 'men', 'will', 'love', 'it', 'for', 'its', 'spot', '-', 'on', 'father', '/', 'son', 'character', 'study', '--', 'one', 'great', 'little', 'scene', 'after', 'another', '.', 'And', 'it', 'has', 'just', 'enough', 'of', 'an', 'edge', 'to', 'be', 'a', 'true', 'indie', 'find', '.', 'Obviously', 'this', 'is', 'a', 'labor', 'of', 'love', 'for', 'Paul', 'Reiser', 'who', 'understands', 'what', 'it', \"'s\", 'like', 'to', 'be', 'both', 'a', 'father', 'and', 'a', 'son', ',', 'as', 'well', 'as', 'to', 'have', 'both', 'laughter', 'and', 'tears', 'as', 'you', 'move', 'through', 'life', '.', 'The', 'most', 'fun', 'part', ',', 'though', ',', 'was', 'watching', 'Reiser', 'watch', 'Falk', '.', 'You', 'could', 'tell', 'it', 'was', 'both', 'his', 'character', 'coming', 'to', 'a', 'new', 'appreciation', 'of', 'his', 'father', 'and', 'a', 'fellow', 'actor', 'really', 'enjoying', 'Peter', 'Falk', \"'s\", 'special', 'craft', '.', 'Really', 'delightful', '.', 'Let', \"'s\", 'hope', 'this', 'film', 'makes', 'it', 'into', 'theaters', 'around', 'the', 'country', 'sometime', 'soon', 'so', 'everyone', 'can', 'have', 'a', 'chance', 'to', 'laugh', 'and', 'cry', 'with', 'Paul', 'Reiser', 'and', 'folks', '.'], 'label': 'pos'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HMVqiZd6tR0"
      },
      "source": [
        "import random\n",
        "\n",
        "train_data, valid_data = train_data.split(random_state = random.seed(SEED))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uOeQ6KpP7M-0",
        "outputId": "4c59e9ef-cb49-46ca-a25a-af3ad5df8921"
      },
      "source": [
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of validation examples: {len(valid_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples: 17500\n",
            "Number of validation examples: 7500\n",
            "Number of testing examples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KixkM1jQ7TB-"
      },
      "source": [
        "MAX_VOCAB_SIZE = 25_000\n",
        "\n",
        "TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hD4SFKnc7g0D",
        "outputId": "c91a096d-0053-49e0-a176-1de0c8b66d89"
      },
      "source": [
        "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n",
        "print(f\"Unique tokens in LABEL vocabulary: {len(LABEL.vocab)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique tokens in TEXT vocabulary: 25002\n",
            "Unique tokens in LABEL vocabulary: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ttKvFTCQ7isK",
        "outputId": "257086e3-d9e8-4729-afd4-018a7acf734b"
      },
      "source": [
        "print(TEXT.vocab.freqs.most_common(20))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('the', 202995), (',', 192589), ('.', 166117), ('a', 109823), ('and', 109606), ('of', 100631), ('to', 94086), ('is', 76261), ('in', 61475), ('I', 54572), ('it', 53798), ('that', 49288), ('\"', 44123), (\"'s\", 43366), ('this', 42619), ('-', 37290), ('/><br', 35673), ('was', 35263), ('as', 30534), ('with', 29957)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fZXIsIV47mlI",
        "outputId": "2723bd84-14f5-41ee-9463-66b079ddfdf2"
      },
      "source": [
        "print(TEXT.vocab.itos[:10])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<unk>', '<pad>', 'the', ',', '.', 'a', 'and', 'of', 'to', 'is']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vmbx3T9-7x4g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6a37daf-14fe-4bc0-a67b-5c1b6812cfdc"
      },
      "source": [
        "print(LABEL.vocab.stoi)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(None, {'neg': 0, 'pos': 1})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3gBfP6mEJ_0"
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    device = device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2ZQQV1-ELZf"
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class LSTM(nn.Module):\n",
        "    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
        "        \n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=1)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "        \n",
        "    def forward(self, text):\n",
        "\n",
        "        #text = [sent len, batch size]\n",
        "        \n",
        "        embedded = self.embedding(text)\n",
        "        \n",
        "        #embedded = [sent len, batch size, emb dim]\n",
        "        \n",
        "        output, (hidden, cn) = self.lstm(embedded)\n",
        "        \n",
        "        #output = [sent len, batch size, hid dim]\n",
        "        #hidden = [1, batch size, hid dim]\n",
        "        hidden = hidden[-1,:,:]\n",
        "        # import pdb\n",
        "        # pdb.set_trace()\n",
        "        # output, (hidden, b) = self.lstm(embedded)\n",
        "        assert torch.equal(output[-1,:,:], hidden.squeeze(0))\n",
        "        \n",
        "        return self.fc(hidden.squeeze(0))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x0_X5kSwENad"
      },
      "source": [
        "\n",
        "INPUT_DIM = len(TEXT.vocab)\n",
        "EMBEDDING_DIM = 100\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1\n",
        "\n",
        "model = LSTM(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdGb8dKBEO2x",
        "outputId": "b71bbf47-bc73-4121-98e0-734c6b13fed5"
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 2,867,049 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAeEtXiJEQCj"
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "optimizer = optim.Adam(model.parameters())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Utp4-qAERRG"
      },
      "source": [
        "criterion = nn.BCEWithLogitsLoss()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyAXf58FESdL"
      },
      "source": [
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4yNiGXQETh9"
      },
      "source": [
        "def binary_accuracy(preds, y):\n",
        "    \"\"\"\n",
        "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
        "    \"\"\"\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1iGJW1wEUrL"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        # import pdb\n",
        "        # pdb.set_trace()\n",
        "                \n",
        "        predictions = model(batch.text).squeeze(1)\n",
        "        \n",
        "        loss = criterion(predictions, batch.label)\n",
        "        \n",
        "        acc = binary_accuracy(predictions, batch.label)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNQxQS3tEWUW"
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            predictions = model(batch.text).squeeze(1)\n",
        "            \n",
        "            loss = criterion(predictions, batch.label)\n",
        "            \n",
        "            acc = binary_accuracy(predictions, batch.label)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DVM8MtV6EYIw"
      },
      "source": [
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJ5KZmM4EZXW",
        "outputId": "f7308315-dc34-467d-f983-88ef78f10657"
      },
      "source": [
        "N_EPOCHS = 10\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "    \n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'tut1-model.pt')\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.692 | Train Acc: 50.43%\n",
            "\t Val. Loss: 0.699 |  Val. Acc: 50.15%\n",
            "Epoch: 02 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.691 | Train Acc: 50.55%\n",
            "\t Val. Loss: 0.696 |  Val. Acc: 51.26%\n",
            "Epoch: 03 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.690 | Train Acc: 50.79%\n",
            "\t Val. Loss: 0.708 |  Val. Acc: 51.91%\n",
            "Epoch: 04 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.688 | Train Acc: 50.70%\n",
            "\t Val. Loss: 0.711 |  Val. Acc: 52.45%\n",
            "Epoch: 05 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.689 | Train Acc: 50.29%\n",
            "\t Val. Loss: 0.707 |  Val. Acc: 52.01%\n",
            "Epoch: 06 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.688 | Train Acc: 50.91%\n",
            "\t Val. Loss: 0.715 |  Val. Acc: 51.31%\n",
            "Epoch: 07 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.687 | Train Acc: 51.00%\n",
            "\t Val. Loss: 0.728 |  Val. Acc: 53.83%\n",
            "Epoch: 08 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.687 | Train Acc: 50.96%\n",
            "\t Val. Loss: 0.735 |  Val. Acc: 52.26%\n",
            "Epoch: 09 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.686 | Train Acc: 51.09%\n",
            "\t Val. Loss: 0.772 |  Val. Acc: 52.60%\n",
            "Epoch: 10 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.640 | Train Acc: 61.82%\n",
            "\t Val. Loss: 0.892 |  Val. Acc: 52.45%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qIiKAJMaEbKO",
        "outputId": "eb1da8b2-c963-4944-9e4d-c8957147a051"
      },
      "source": [
        "model.load_state_dict(torch.load('tut1-model.pt'))\n",
        "\n",
        "test_loss, test_acc = evaluate(model, test_iterator, criterion)\n",
        "\n",
        "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.724 | Test Acc: 56.23%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G024NssCEcj0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}