nlp_pytorch_3_mlp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "from torch import nn \n",
    "import numpy as np "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(1400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.rand(2, 3) # , 5)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.sigmoid(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.relu(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_out = torch.sum(x, dim=0) # dim 1 , 2 , ..? \n",
    "x_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ softmax(x)_i = \\frac{exp(x_i)}{\\sum_{j}^{ }exp(x_j))}.$$\n",
    "\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.softmax(x, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = nn.Linear(3, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_l1 = l1(x)\n",
    "result_l1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_l1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# ⚠️⚠️⚠️\n",
    "\n",
    "####   • torch.nn only supports mini-batches. The entire torch.nn package only supports inputs that are a mini-batch of samples, and not a single sample.\n",
    "\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- pip install torchtext --user -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.datasets import AG_NEWS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- AG is a collection of more than 1 million news articles\n",
    "- Consists of class ids 1-4 where 1-World, 2-Sports, 3-Business, 4-Sci/Tech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter = AG_NEWS(root='${HOME}/.data', split='train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "next(train_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data.utils import get_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Also look at nltk, spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_data = ['a', 'a', 'b', 'c', 'a', 'c', 'd', 'd']\n",
    "sample_counter = Counter(sample_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = [i[1] for i in AG_NEWS(split='train')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = [i[0] for i in AG_NEWS(split='train')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = get_tokenizer('basic_english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer(train_x[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NLTK\n",
    "\n",
    "- pip install nltk\n",
    "- python shell for downloading data:\n",
    "\n",
    "    \\>>> import nltk \n",
    "\n",
    "    \\>>> nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = set()\n",
    "for line in train_x:\n",
    "    all_words.update(tokenizer(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(list(all_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "class CountEncoder:\n",
    "    def __init__(self, string_data: List[str], min_count = 12): \n",
    "        self.all_words = Counter()\n",
    "        self.trans_dict = str.maketrans('', '', string.punctuation)\n",
    "        for line in string_data:\n",
    "            tokenized_line = tokenizer(line.lower())\n",
    "            line = [l.translate(self.trans_dict) for l in tokenized_line]\n",
    "            self.all_words.update(line)\n",
    "        for punc in stopwords.words('english'):\n",
    "            del self.all_words[punc]\n",
    "        for c in list(self.all_words):\n",
    "            if self.all_words[c] < min_count:\n",
    "                del self.all_words[c]\n",
    "        self.word_count = len(self.all_words)\n",
    "        self.all_words = {key:index for index, key in enumerate(self.all_words)}\n",
    "        \n",
    "    def translate(self, strings):\n",
    "        string_len = len(strings)\n",
    "        output = sparse.dok_matrix((string_len, self.word_count), dtype=np.float32)\n",
    "        for index, s in enumerate(strings):\n",
    "            s = s.lower()\n",
    "            tokenized_line = tokenizer(s)\n",
    "            line = [l.translate(self.trans_dict) for l in tokenized_line]\n",
    "            line = [l for l in tokenized_line if l in self.all_words]\n",
    "            counter = Counter(line)\n",
    "            for c in counter.keys():\n",
    "                output[index, self.all_words[c]] = counter[c]\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_encoder = CountEncoder(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_encoder.word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x_sparse = count_encoder.translate(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CounterModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim = 48, output_dim = 4):\n",
    "        super().__init__()\n",
    "        self.w1 = nn.Linear(input_dim, hidden_dim)\n",
    "        self.w2 = nn.Linear(hidden_dim, output_dim)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.w1(x))\n",
    "        x = self.w2(x) # why we are not using softmax here??\n",
    "        return x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CounterModel(count_encoder.word_count).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 256\n",
    "epoches = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train loop\n",
    "for e in range(epoches):\n",
    "    size = len(train_x) // batch_size\n",
    "    for i in range(0, len(train_x), batch_size):\n",
    "        batch_x = torch.from_numpy(train_x_sparse[i:i+batch_size].toarray()).to(device)\n",
    "        batch_y = torch.tensor(train_y[i: i+batch_size], dtype=torch.int64).to(device) - 1\n",
    "        pred = model(batch_x)\n",
    "        loss_val = loss_fn(pred, batch_y)\n",
    "        optimizer.zero_grad()\n",
    "        loss_val.backward()\n",
    "        optimizer.step()\n",
    "        if i%100 == 0:\n",
    "            loss = loss_val.item()\n",
    "            print(f\"loss: {loss:>7f}  epoch: {e}, batch_percent: [{i/len(train_x)*100}%]\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.tensor(train_y, dtype=torch.int64).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x = [i[1] for i in AG_NEWS(split='test')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_y = [i[0] for i in AG_NEWS(split='test')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x_sparse = count_encoder.translate(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x_sparse[:100].toarray().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    test_batch = 1000\n",
    "    test_x_batch = torch.from_numpy(test_x_sparse[:test_batch].toarray()).to(device)\n",
    "    test_y_batch = torch.tensor(test_y[:test_batch]).to(device)\n",
    "    outputs = model(test_x_batch)\n",
    "    _, predict = torch.max(outputs, dim=1)\n",
    "    correct = (predict == test_y_batch).sum().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}