transformers.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_GdgF0alssCh"
      },
      "source": [
        "# Implementing Transformer\n",
        "\n",
        "## What's attention?\n",
        "\n",
        "- Attention was introduced in the Attention is all you need paper, for a translation task\n",
        "- Attention is a small part of the network.\n",
        "- GPT is a generative network which uses only the decoder part of the model presented in Attention is all you need.\n",
        "- BERT is also a transformer model, that can be finetuned for many tasks\n",
        "\n",
        "Let's implement the translation model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVQhhGihuXQx"
      },
      "source": [
        "# Let's start with BERT and transformers library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "from torch.utils.tensorboard import SummaryWriter\n",
        "\n",
        "writer = SummaryWriter(comment=\"unfreezed BERT 128\")\n",
        "PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0\n",
        "PYTORCH_ENABLE_MPS_FALLBACK=1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        },
        "id": "7DNY8osmuqnW",
        "outputId": "5a384097-d53f-4ca5-f8e0-ec53609e56b2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of training sentences: 1,121\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>task_id</th>\n",
              "      <th>text</th>\n",
              "      <th>AI</th>\n",
              "      <th>code</th>\n",
              "      <th>news</th>\n",
              "      <th>phospho</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>758</th>\n",
              "      <td>ae6d1ebcf2624338903d87ca98bce344</td>\n",
              "      <td>Tesla vs Rivian ### Tesla Vehicle Models for 2...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>770</th>\n",
              "      <td>b1e9c1dd0b514165a96511b41b80f1ff</td>\n",
              "      <td>\\tVente de détail\\t\\tManufacture\\t\\tConsructio...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>160</th>\n",
              "      <td>2400f07c536f4064992eafff5e684444</td>\n",
              "      <td>can i iframe only a specific part of a website...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>672</th>\n",
              "      <td>9a357544f52849028b1e253877ec6cb2</td>\n",
              "      <td>3 RUE DE LA BIENFAISANCE – 28000 The address \"...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>380</th>\n",
              "      <td>511530eca41d46858d8fb85c6fab6842</td>\n",
              "      <td>fais la meme chose pour les paragraphes suivan...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>340ec48ee46643e7817c6c3ecceb1a1f</td>\n",
              "      <td>trouve moi les deux articles de recherche à l'...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130</th>\n",
              "      <td>1beb899acfc544aebb2b2859b3d00df9</td>\n",
              "      <td>reformule mieux : Hétérogénéité spatiale dans ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1017</th>\n",
              "      <td>e80faf0f3757440dac79dd365ccf51a5</td>\n",
              "      <td>Quels sont les points clés de ta politique de ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>2425ba21def74455a04565e1aa063d65</td>\n",
              "      <td>Tu peux créer des images ? Je ne suis pas capa...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>e03dac0d3413476c8f2527e735cd5997</td>\n",
              "      <td>peux tu me donner les inforamtions que tu as s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                               task_id  \\\n",
              "758   ae6d1ebcf2624338903d87ca98bce344   \n",
              "770   b1e9c1dd0b514165a96511b41b80f1ff   \n",
              "160   2400f07c536f4064992eafff5e684444   \n",
              "672   9a357544f52849028b1e253877ec6cb2   \n",
              "380   511530eca41d46858d8fb85c6fab6842   \n",
              "241   340ec48ee46643e7817c6c3ecceb1a1f   \n",
              "130   1beb899acfc544aebb2b2859b3d00df9   \n",
              "1017  e80faf0f3757440dac79dd365ccf51a5   \n",
              "162   2425ba21def74455a04565e1aa063d65   \n",
              "982   e03dac0d3413476c8f2527e735cd5997   \n",
              "\n",
              "                                                   text  AI  code  news  \\\n",
              "758   Tesla vs Rivian ### Tesla Vehicle Models for 2...   0     0     0   \n",
              "770   \\tVente de détail\\t\\tManufacture\\t\\tConsructio...   0     1     0   \n",
              "160   can i iframe only a specific part of a website...   0     1     0   \n",
              "672   3 RUE DE LA BIENFAISANCE – 28000 The address \"...   0     0     0   \n",
              "380   fais la meme chose pour les paragraphes suivan...   0     0     0   \n",
              "241   trouve moi les deux articles de recherche à l'...   0     0     0   \n",
              "130   reformule mieux : Hétérogénéité spatiale dans ...   0     0     0   \n",
              "1017  Quels sont les points clés de ta politique de ...   0     0     0   \n",
              "162   Tu peux créer des images ? Je ne suis pas capa...   0     0     0   \n",
              "982   peux tu me donner les inforamtions que tu as s...   0     0     0   \n",
              "\n",
              "      phospho  \n",
              "758         0  \n",
              "770         0  \n",
              "160         0  \n",
              "672         0  \n",
              "380         0  \n",
              "241         0  \n",
              "130         0  \n",
              "1017        1  \n",
              "162         0  \n",
              "982         0  "
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"/Users/kyoto/git/phospho_project/data.csv\")\n",
        "\n",
        "# Report the number of sentences.\n",
        "print(\"Number of training sentences: {:,}\\n\".format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326,
          "referenced_widgets": [
            "754aefc416214b108c92856d18340da5",
            "fd1c91fa5d074d8abb7dd15b7ef81c20",
            "1d94813b9d2b400db2595141b7373b10",
            "6be0dad82656466099102dddb64119ea",
            "f642ce3da3b349439feb9fb9118e4c05",
            "b0cbab21c872461084786fbea38a2094",
            "abbd8726472443eebb9365334eb634e7",
            "7f7882fbe9d04c7b95068d9a58089650",
            "36717e7f4fbc452eb7d454fdc6633bef",
            "d7faf225275d4dea8adbf7ebd8adf7b9",
            "9b77b73d54af4549bfaec184394e584a",
            "ed8b133fed824ce9bed8a8f64384379d",
            "77e128871ed6448184b7bcd8a7a483db",
            "53bfe556bfe841fd95a24e3563792591",
            "4940afd924ad4d37b2c45ba64f75a5be",
            "86a7ccd61ab545c8a2f5e63101155e79",
            "f526f21fe01443399f044811b54a99c9",
            "ce74d6aa114144a196519687ef1a62fe",
            "76e56a942b3144e2b361858cede0e4f9",
            "63b828c4e9c74f559ce85f5c88beef10",
            "3daf6db764bf40df8fb00f3af0308f05",
            "5247c5098eab491db5b9aad9c58aee41",
            "85d1bb322b564edf90754a08d2e7be9d",
            "bd1fa2ec11d446aa9ad3952c8b35bf31",
            "5f83b98ae18a46b1ae8b0a9a50e4c10f",
            "c423e62fa75e4b4189012c9ef12005a6",
            "163afe786ef5406d86702405fc35c871",
            "cbd57b4ad48b4c8ca4b5616005fce6e7",
            "3ee928be005c4d70b43b7f72d58672d9",
            "2f479528dc7a4941bb4f899aad1619ed",
            "c6262084f781434fba9a273e0a606fff",
            "9eb53e446d9f4b7dae919efb9ec3451d",
            "b836e939b1dc4762bec490893faac335",
            "efcaafee43ea47848fed5aad4e2fd5bd",
            "85999c408377499ba4a6bfee4ff42479",
            "d3fd1b76eb6c48a9aa2c7216ac1032e2",
            "780ba8f6d5c94c66b8f2b617cb387974",
            "9d723a4d5d3a4c6db1d5bfa1ee5ea408",
            "d9a83ce852d64639a8460152def0bef5",
            "111c5c152fc04a6985b345032e2b836b",
            "31e51667f892458db9264f68218c0954",
            "e79a0a56c6cd46c78fddeb532909be44",
            "69962c7207064776b382029883ef3017",
            "45efd2abdefa437998b35130e9fa6a3b"
          ]
        },
        "id": "qZh0J2htu3IC",
        "outputId": "b11e5c16-8106-494c-f524-be8095559d5c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/kyoto/miniforge3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ]
        }
      ],
      "source": [
        "from transformers import BertTokenizer, AutoTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print(\"Loading BERT tokenizer...\")\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "#tokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-small\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BkxDYlm-u4nS",
        "outputId": "7f253861-d493-4b11-e08e-25e8348d1e08"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/Users/kyoto/miniforge3/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2699: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Original:  que faire a new york ce week end?  Voici quelques activités intéressantes à faire à New York ce week-end :\n",
            "\n",
            "1. **Randy Rainbow for President**\n",
            "   - **Description** : Assistez à un spectacle humoristique de Randy Rainbow au Beacon Theater.\n",
            "   - **Date** : Vendredi soir.\n",
            "   - **Lieu** : Beacon Theater, 2124 Broadway, Manhattan.\n",
            "   - [Plus d'infos](https://www.nytimes.com/article/things-to-do-in-nyc.html)\n",
            "   - ![Image](https://i2.wp.com/totravelandbeyond.com/wp-content/uploads/2016/02/things-to-do-in-new-york-city.jpg?resize=565,847)\n",
            "\n",
            "2. **Frieze New York**\n",
            "   - **Description** : Pour les amateurs d'art, visitez Frieze New York, une exposition d'art contemporain avec plus de 1 000 artistes et 200 galeries internationales.\n",
            "   - **Date** : Du 1er au 5 mai 2024.\n",
            "   - **Lieu** : The Shed, Hudson Yards.\n",
            "   - [Plus d'infos](https://loving-newyork.com/new-york-in-may/)\n",
            "   - ![Image](https://media.timeout.com/images/103495347/image.jpg)\n",
            "\n",
            "3. **Fleet Week NYC 2024**\n",
            "   - **Description** : Célébrez la Fleet Week, une semaine d'événements honorant les membres de la Marine, de la Garde côtière et du Corps des Marines des États-Unis.\n",
            "   - **Date** : Du 22 au 28 mai 2024.\n",
            "   - **Lieu** : Divers lieux à travers la ville.\n",
            "   - [Plus d'infos](https://www.timeout.com/newyork/events-calendar/may-events-calendar)\n",
            "   - ![Image](https://cdn.fodors.com/wp-content/uploads/2016/02/2-Times-Square.jpg)\n",
            "\n",
            "4. **Cinco de Mayo Celebrations**\n",
            "   - **Description** : Participez aux célébrations de Cinco de Mayo, une fête commémorant la victoire du Mexique contre la France en 1862.\n",
            "   - **Date** : 5 mai 2024.\n",
            "   - **Lieu** : Divers lieux à travers la ville.\n",
            "   - [Plus d'infos](https://secretnyc.co/things-to-do-may-nyc/)\n",
            "   - ![Image](https://www.thedatingdivas.com/wp-content/uploads/2021/10/Things-to-Do-In-NYC.jpg)\n",
            "\n",
            "5. **NYC Craft Beer Fest**\n",
            "   - **Description** : Profitez du NYC Craft Beer Fest à Chelsea, une occasion unique de déguster une large sélection de bières du monde entier.\n",
            "   - **Date** : 9 mars 2024 (à vérifier pour les dates spécifiques de mai).\n",
            "   - **Lieu** : Chelsea.\n",
            "   - [Plus d'infos](https://loving-newyork.com/things-to-do/things-to-do-in-nyc-this-weekend/)\n",
            "   - ![Image](https://www.nyrush.com/wp-content/uploads/2021/08/Untitled-design-66.jpg)\n",
            "\n",
            "Profitez bien de votre week-end à New York !\n",
            "Token IDs: tensor([  101, 10861,  4189,  2063,  1037,  2047,  2259,  8292,  2733,  2203,\n",
            "         1029, 29536, 28775, 10861,  2140, 10997,  2552, 12848,  7616,  6970,\n",
            "         7971, 24985,  1037,  4189,  2063,  1037,  2047,  2259,  8292,  2733,\n",
            "         1011,  2203,  1024,  1015,  1012,  1008,  1008,  9744, 10098,  2005,\n",
            "         2343,  1008,  1008,  1011,  1008,  1008,  6412,  1008,  1008,  1024,\n",
            "         6509,  9351,  1037,  4895, 21177,  8562,  2923,  7413,  2139,  9744,\n",
            "        10098,  8740, 14400,  4258,  1012,  1011,  1008,  1008,  3058,  1008,\n",
            "         1008,  1024,  2310,  4859,  5596,  2072,  2061,  4313,  1012,  1011,\n",
            "         1008,  1008, 22470,  1008,  1008,  1024, 14400,  4258,  1010, 18164,\n",
            "         2549,  5934,  1010,  7128,  1012,  1011,  1031,  4606,  1040,  1005,\n",
            "        18558,  2015,  1033,  1006, 16770,  1024,  1013,  1013,  7479,  1012,\n",
            "         6396,  7292,  2015,  1012,  4012,  1013,  3720,  1013,  2477,  1011,\n",
            "         2000,  1011,  2079,  1011,  1999,  1011, 16392,   102])\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in df[\"text\"]:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "        sent,  # Sentence to encode.\n",
        "        add_special_tokens=True,  # Add '[CLS]' and '[SEP]'\n",
        "        max_length=128,  # Pad & truncate all sentences.\n",
        "        pad_to_max_length=True,\n",
        "        return_attention_mask=True,  # Construct attn. masks.\n",
        "        return_tensors=\"pt\",  # Return pytorch tensors.\n",
        "    )\n",
        "\n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_dict[\"input_ids\"])\n",
        "\n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict[\"attention_mask\"])\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = df[[\"AI\", \"code\", \"news\", \"phospho\"]].values\n",
        "labels_tensor = torch.tensor(labels, dtype=torch.float32)\n",
        "\n",
        "mps_device = torch.device(\"mps\")\n",
        "\n",
        "input_ids.to(mps_device)\n",
        "labels_tensor.to(mps_device)\n",
        "\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print(\"Original: \", df[\"text\"][0])\n",
        "print(\"Token IDs:\", input_ids[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w00HlDIlvPuw",
        "outputId": "996ff045-ea70-4fad-fd70-5850d64f9625"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1,008 training samples\n",
            "  113 validation samples\n"
          ]
        }
      ],
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels_tensor)\n",
        "# Create a 90-10 train-validation split.\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "# Divide the dataset by randomly selecting samples.\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print(\"{:>5,} training samples\".format(train_size))\n",
        "print(\"{:>5,} validation samples\".format(val_size))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "xWRPzg4kvQUB"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it\n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch\n",
        "# size of 16 or 32.\n",
        "batch_size = 8\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order.\n",
        "train_dataloader = DataLoader(\n",
        "    train_dataset,  # The training samples.\n",
        "    sampler=RandomSampler(train_dataset),  # Select batches randomly\n",
        "    batch_size=batch_size,  # Trains with this batch size.\n",
        ")\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "    val_dataset,  # The validation samples.\n",
        "    sampler=SequentialSampler(val_dataset),  # Pull out batches sequentially.\n",
        "    batch_size=batch_size,  # Evaluate with this batch size.\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379,
          "referenced_widgets": [
            "cfab8cbfe5fa4f2087a73e697fd59b5a",
            "4eb5f692a9a14186bc0dafa12561073d",
            "de321fe6c04c4b0eb2cf25d14f0b73d4",
            "4d8c3670f82949c0997b29f7543b1482",
            "1bcf03fa396b4761b9a29b8557124a69",
            "086205e94683476198aaa96b9e8535d2",
            "8f772eca26164434819314a6c67e47d0",
            "6becf7082d3a4b0da24be1b9fae2ab2d",
            "14fef336c7204910a8608f51da1f6505",
            "419c5d3575324a148203960eb9483c87",
            "5774d4df96b3476a9a26506e23f8a682"
          ]
        },
        "id": "OZ9qwP1OwCeX",
        "outputId": "c78b2ca2-d15e-4e05-995d-7ac948d4d5f3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BERTClass(\n",
              "  (bert_model): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (linear): Linear(in_features=768, out_features=4, bias=True)\n",
              ")"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import (\n",
        "    AdamW,\n",
        "    BertModel,\n",
        "    AutoModelForSeq2SeqLM,\n",
        "    BertForSequenceClassification\n",
        ")\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "\n",
        "### OG BERT MODEL\n",
        "\n",
        "class BERTClass(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BERTClass, self).__init__()\n",
        "        self.bert_model = BertModel.from_pretrained('bert-base-uncased', return_dict=True)\n",
        "        self.dropout = torch.nn.Dropout(0.1)\n",
        "        self.linear = torch.nn.Linear(768, 4)\n",
        "        #self.bert_model.requires_grad_ = False   #freezing the gradients or not\n",
        "        \n",
        "    def forward(self, input_ids, attn_mask, token_type_ids):\n",
        "        output = self.bert_model(\n",
        "            input_ids, attention_mask=attn_mask, token_type_ids=token_type_ids\n",
        "        )\n",
        "        output_dropout = self.dropout(output.pooler_output)\n",
        "        output = self.linear(output_dropout)\n",
        "        return output\n",
        "\n",
        "model = BERTClass()\n",
        "\n",
        "### T5 MODEL\n",
        "\n",
        "# class T5Model(torch.nn.Module):\n",
        "#     def __init__(self):\n",
        "#         super(T5Model, self).__init__()\n",
        "#         self.t5_model = AutoModelForSeq2SeqLM.from_pretrained(\"google/flan-t5-small\")\n",
        "#         self.dropout = torch.nn.Dropout(0.1)\n",
        "#         self.linear = torch.nn.Linear(768, 4)\n",
        "#         self.t5_model.requires_grad = False\n",
        "\n",
        "#     def forward(self, input_ids, attn_mask, token_type_ids):\n",
        "#         output = self.t5_model(\n",
        "#             input_ids=input_ids,\n",
        "#             attention_mask=attn_mask,\n",
        "#             decoder_input_ids=input_ids,\n",
        "#             output_hidden_states=True\n",
        "#         )\n",
        "#         output_dropout = self.dropout(output.encoder_hidden_states[-1][:,0,:])\n",
        "#         output = self.linear(output_dropout)\n",
        "#         return output\n",
        "\n",
        "\n",
        "\n",
        "### BERT Seq Model\n",
        "#model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=4)\n",
        "mps_device = torch.device(\"mps\")\n",
        "model.to(mps_device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "nDjwqjjTKz-0"
      },
      "outputs": [],
      "source": [
        "def loss_fn(outputs, targets):\n",
        "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "94WBLh8VwCyq"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/kyoto/miniforge3/lib/python3.10/site-packages/transformers/optimization.py:588: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch)\n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(\n",
        "    model.parameters(),\n",
        "    lr=2e-5,  # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "    eps=1e-8,  # args.adam_epsilon  - default is 1e-8.\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "qBwsJx9Xwa9P"
      },
      "outputs": [],
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4.\n",
        "# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
        "# training data.\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs].\n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer,\n",
        "    num_warmup_steps=0,  # Default value in run_glue.py\n",
        "    num_training_steps=total_steps,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "dZjqHl5awyVA"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Apple GPU\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.backends\n",
        "import torch.backends.mps\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():\n",
        "    # Tell PyTorch to use the GPU.\n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print(\"There are %d GPU(s) available.\" % torch.cuda.device_count())\n",
        "\n",
        "    print(\"We will use the GPU:\", torch.cuda.get_device_name(0))\n",
        "\n",
        "elif torch.backends.mps.is_available():\n",
        "    print(\"Apple GPU\")\n",
        "    device = torch.device(\"mps\")\n",
        "else:\n",
        "    print(\"No GPU available, using the CPU instead.\")\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/kyoto/miniforge3/lib/python3.10/site-packages/transformers/training_args.py:2101: UserWarning: `use_mps_device` is deprecated and will be removed in version 5.0 of 🤗 Transformers. `mps` device will be used by default if available similar to the way `cuda` device is used.Therefore, no action from user is required. \n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import TrainingArguments\n",
        "training_args = TrainingArguments(output_dir=\"test_trainer\", use_mps_device=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qVUZSuCZwbQT",
        "outputId": "00330ecd-86f4-4dff-d394-f3c00e1947a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    126.    Elapsed: 9.41854214668274.\n",
            "  Batch    80  of    126.    Elapsed: 17.989113807678223.\n",
            "  Batch   120  of    126.    Elapsed: 26.552665948867798.\n",
            "\n",
            "  Average training loss: 0.3459\n",
            "  Training epoch took: 27.83442783355713\n",
            "\n",
            "Running Validation...\n",
            "  F1 Score: 0.31\n",
            "  Recall Score: 0.28\n",
            "  Precision Score: 0.37\n",
            " Multilabel Confusion Matrix: [[[101   0]\n",
            "  [ 12   0]]\n",
            "\n",
            " [[101   1]\n",
            "  [  5   6]]\n",
            "\n",
            " [[ 98   0]\n",
            "  [ 15   0]]\n",
            "\n",
            " [[103   1]\n",
            "  [  2   7]]]\n",
            "  Accuracy: 0.93\n",
            "  Validation Loss: 0.2317\n",
            "  Validation took: 0.8545770645141602\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/kyoto/miniforge3/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1517: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Batch    40  of    126.    Elapsed: 8.566675901412964.\n",
            "  Batch    80  of    126.    Elapsed: 17.1368670463562.\n",
            "  Batch   120  of    126.    Elapsed: 25.759382009506226.\n",
            "\n",
            "  Average training loss: 0.1953\n",
            "  Training epoch took: 27.051017999649048\n",
            "\n",
            "Running Validation...\n",
            "  F1 Score: 0.70\n",
            "  Recall Score: 0.60\n",
            "  Precision Score: 0.93\n",
            " Multilabel Confusion Matrix: [[[101   0]\n",
            "  [  7   5]]\n",
            "\n",
            " [[101   1]\n",
            "  [  3   8]]\n",
            "\n",
            " [[ 97   1]\n",
            "  [  9   6]]\n",
            "\n",
            " [[104   0]\n",
            "  [  0   9]]]\n",
            "  Accuracy: 0.96\n",
            "  Validation Loss: 0.1548\n",
            "  Validation took: 0.757655143737793\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    126.    Elapsed: 8.677013874053955.\n",
            "  Batch    80  of    126.    Elapsed: 17.277265787124634.\n",
            "  Batch   120  of    126.    Elapsed: 25.84067678451538.\n",
            "\n",
            "  Average training loss: 0.1242\n",
            "  Training epoch took: 27.1249737739563\n",
            "\n",
            "Running Validation...\n",
            "  F1 Score: 0.72\n",
            "  Recall Score: 0.64\n",
            "  Precision Score: 0.92\n",
            " Multilabel Confusion Matrix: [[[100   1]\n",
            "  [  4   8]]\n",
            "\n",
            " [[100   2]\n",
            "  [  3   8]]\n",
            "\n",
            " [[ 98   0]\n",
            "  [ 10   5]]\n",
            "\n",
            " [[104   0]\n",
            "  [  0   9]]]\n",
            "  Accuracy: 0.96\n",
            "  Validation Loss: 0.1342\n",
            "  Validation took: 0.7533559799194336\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    126.    Elapsed: 8.563272953033447.\n",
            "  Batch    80  of    126.    Elapsed: 17.13568615913391.\n",
            "  Batch   120  of    126.    Elapsed: 25.696732997894287.\n",
            "\n",
            "  Average training loss: 0.0954\n",
            "  Training epoch took: 26.981160163879395\n",
            "\n",
            "Running Validation...\n",
            "  F1 Score: 0.77\n",
            "  Recall Score: 0.72\n",
            "  Precision Score: 0.82\n",
            " Multilabel Confusion Matrix: [[[ 99   2]\n",
            "  [  3   9]]\n",
            "\n",
            " [[100   2]\n",
            "  [  4   7]]\n",
            "\n",
            " [[ 95   3]\n",
            "  [  6   9]]\n",
            "\n",
            " [[104   0]\n",
            "  [  0   9]]]\n",
            "  Accuracy: 0.96\n",
            "  Validation Loss: 0.1196\n",
            "  Validation took: 0.7545089721679688\n",
            "\n",
            "Training complete!\n",
            "Total training took 112.11344480514526 (h:mm:ss)\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "import numpy as np\n",
        "import time\n",
        "from sklearn.metrics import f1_score, recall_score, precision_score, multilabel_confusion_matrix\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss,\n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "total_eval_f1 = 0\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    if pred_flat.shape[0] != labels_flat.shape[0]:\n",
        "        raise ValueError(\n",
        "            f\"Predicted labels and true labels are not the same length: {pred_flat.shape[0]} vs {labels_flat.shape[0]}\"\n",
        "        )\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "\n",
        "def calculate_metrics(logits, labels):\n",
        "    # Assuming logits and labels are PyTorch tensors when this function is called\n",
        "    preds = torch.sigmoid(logits) > 0.5\n",
        "    preds_flat = preds.view(-1)\n",
        "    labels_flat = labels.view(-1)\n",
        "\n",
        "    # Calculate accuracy and F1 score\n",
        "    accuracy = (preds_flat == labels_flat).float().mean().item()\n",
        "    return accuracy\n",
        "\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    all_preds = []\n",
        "    all_labels = []\n",
        "\n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"======== Epoch {:} / {:} ========\".format(epoch_i + 1, epochs))\n",
        "    print(\"Training...\")\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to\n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = time.time() - t0\n",
        "\n",
        "            # Report progress.\n",
        "            print(\n",
        "                \"  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.\".format(\n",
        "                    step, len(train_dataloader), elapsed\n",
        "                )\n",
        "            )\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the\n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because\n",
        "        # accumulating the gradients is \"convenient while training RNNs\".\n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()\n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # The documentation for this `model` function is here:\n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        # It returns different numbers of parameters depending on what arguments\n",
        "        # arge given and what flags are set. For our useage here, it returns\n",
        "        # the loss (because we provided labels) and the \"logits\"--the model\n",
        "        # outputs prior to activation.\n",
        "        output = model(b_input_ids,token_type_ids=None,attn_mask=b_input_mask)        \n",
        "        logits = output\n",
        "        loss = loss_fn(output, b_labels)\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value\n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)\n",
        "\n",
        "    # Measure how long this epoch took.\n",
        "    training_time = time.time() - t0\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.4f}\".format(avg_train_loss))\n",
        "    writer.add_scalar(\"Loss func\", avg_train_loss, epoch_i)\n",
        "\n",
        "    print(\"  Training epoch took: {:}\".format(training_time))\n",
        "\n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using\n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():\n",
        "            output = model(b_input_ids, attn_mask=b_input_mask, token_type_ids=None)\n",
        "\n",
        "            loss = loss_fn(output, b_labels)\n",
        "\n",
        "            logits = output.detach().cpu()\n",
        "\n",
        "            total_eval_loss += loss.item()\n",
        "\n",
        "            labels = b_labels.detach().cpu()\n",
        "\n",
        "            preds = (torch.sigmoid(logits) > 0.5).int()\n",
        "            accuracy = calculate_metrics(logits, labels)\n",
        "            total_eval_accuracy += accuracy\n",
        "\n",
        "            # Store predictions and labels\n",
        "            all_preds.append(preds)\n",
        "            all_labels.append(labels)\n",
        "\n",
        "    all_preds = torch.cat(all_preds).numpy()\n",
        "    all_labels = torch.cat(all_labels).numpy()\n",
        "\n",
        "    # Calculate F1 score using 'micro' to consider label imbalance\n",
        "    f1 = f1_score(all_labels, all_preds, average=\"weighted\")\n",
        "    print(\"  F1 Score: {:.2f}\".format(f1))\n",
        "\n",
        "    recall = recall_score(all_labels, all_preds, average=\"weighted\")\n",
        "    print(\"  Recall Score: {:.2f}\".format(recall))\n",
        "\n",
        "    precision = precision_score(all_labels, all_preds, average=\"weighted\")\n",
        "    print(\"  Precision Score: {:.2f}\".format(precision))\n",
        "\n",
        "\n",
        "    matrix = multilabel_confusion_matrix(all_labels, all_preds)\n",
        "    print(\" Multilabel Confusion Matrix:\", matrix)\n",
        "\n",
        "    writer.add_scalar(\"F1\", f1, epoch_i)\n",
        "    writer.add_scalar(\"Recall\", recall, epoch_i)\n",
        "    writer.add_scalar(\"Precision\", precision, epoch_i)\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = time.time() - t0\n",
        "\n",
        "    print(\"  Validation Loss: {0:.4f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            \"epoch\": epoch_i + 1,\n",
        "            \"Training Loss\": avg_train_loss,\n",
        "            \"Valid. Loss\": avg_val_loss,\n",
        "            \"Valid. Accur.\": avg_val_accuracy,\n",
        "            \"Training Time\": training_time,\n",
        "            \"Validation Time\": validation_time,\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(time.time() - total_t0))\n",
        "writer.flush()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.14"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "086205e94683476198aaa96b9e8535d2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "111c5c152fc04a6985b345032e2b836b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14fef336c7204910a8608f51da1f6505": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "163afe786ef5406d86702405fc35c871": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bcf03fa396b4761b9a29b8557124a69": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d94813b9d2b400db2595141b7373b10": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f7882fbe9d04c7b95068d9a58089650",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_36717e7f4fbc452eb7d454fdc6633bef",
            "value": 48
          }
        },
        "2f479528dc7a4941bb4f899aad1619ed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31e51667f892458db9264f68218c0954": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36717e7f4fbc452eb7d454fdc6633bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3daf6db764bf40df8fb00f3af0308f05": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ee928be005c4d70b43b7f72d58672d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "419c5d3575324a148203960eb9483c87": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45efd2abdefa437998b35130e9fa6a3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4940afd924ad4d37b2c45ba64f75a5be": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3daf6db764bf40df8fb00f3af0308f05",
            "placeholder": "​",
            "style": "IPY_MODEL_5247c5098eab491db5b9aad9c58aee41",
            "value": " 232k/232k [00:00&lt;00:00, 2.85MB/s]"
          }
        },
        "4d8c3670f82949c0997b29f7543b1482": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_419c5d3575324a148203960eb9483c87",
            "placeholder": "​",
            "style": "IPY_MODEL_5774d4df96b3476a9a26506e23f8a682",
            "value": " 440M/440M [00:04&lt;00:00, 50.5MB/s]"
          }
        },
        "4eb5f692a9a14186bc0dafa12561073d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_086205e94683476198aaa96b9e8535d2",
            "placeholder": "​",
            "style": "IPY_MODEL_8f772eca26164434819314a6c67e47d0",
            "value": "model.safetensors: 100%"
          }
        },
        "5247c5098eab491db5b9aad9c58aee41": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53bfe556bfe841fd95a24e3563792591": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_76e56a942b3144e2b361858cede0e4f9",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_63b828c4e9c74f559ce85f5c88beef10",
            "value": 231508
          }
        },
        "5774d4df96b3476a9a26506e23f8a682": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f83b98ae18a46b1ae8b0a9a50e4c10f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f479528dc7a4941bb4f899aad1619ed",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c6262084f781434fba9a273e0a606fff",
            "value": 466062
          }
        },
        "63b828c4e9c74f559ce85f5c88beef10": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "69962c7207064776b382029883ef3017": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6be0dad82656466099102dddb64119ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7faf225275d4dea8adbf7ebd8adf7b9",
            "placeholder": "​",
            "style": "IPY_MODEL_9b77b73d54af4549bfaec184394e584a",
            "value": " 48.0/48.0 [00:00&lt;00:00, 2.78kB/s]"
          }
        },
        "6becf7082d3a4b0da24be1b9fae2ab2d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "754aefc416214b108c92856d18340da5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fd1c91fa5d074d8abb7dd15b7ef81c20",
              "IPY_MODEL_1d94813b9d2b400db2595141b7373b10",
              "IPY_MODEL_6be0dad82656466099102dddb64119ea"
            ],
            "layout": "IPY_MODEL_f642ce3da3b349439feb9fb9118e4c05"
          }
        },
        "76e56a942b3144e2b361858cede0e4f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77e128871ed6448184b7bcd8a7a483db": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f526f21fe01443399f044811b54a99c9",
            "placeholder": "​",
            "style": "IPY_MODEL_ce74d6aa114144a196519687ef1a62fe",
            "value": "vocab.txt: 100%"
          }
        },
        "780ba8f6d5c94c66b8f2b617cb387974": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69962c7207064776b382029883ef3017",
            "placeholder": "​",
            "style": "IPY_MODEL_45efd2abdefa437998b35130e9fa6a3b",
            "value": " 570/570 [00:00&lt;00:00, 18.9kB/s]"
          }
        },
        "7f7882fbe9d04c7b95068d9a58089650": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85999c408377499ba4a6bfee4ff42479": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9a83ce852d64639a8460152def0bef5",
            "placeholder": "​",
            "style": "IPY_MODEL_111c5c152fc04a6985b345032e2b836b",
            "value": "config.json: 100%"
          }
        },
        "85d1bb322b564edf90754a08d2e7be9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bd1fa2ec11d446aa9ad3952c8b35bf31",
              "IPY_MODEL_5f83b98ae18a46b1ae8b0a9a50e4c10f",
              "IPY_MODEL_c423e62fa75e4b4189012c9ef12005a6"
            ],
            "layout": "IPY_MODEL_163afe786ef5406d86702405fc35c871"
          }
        },
        "86a7ccd61ab545c8a2f5e63101155e79": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f772eca26164434819314a6c67e47d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b77b73d54af4549bfaec184394e584a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d723a4d5d3a4c6db1d5bfa1ee5ea408": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9eb53e446d9f4b7dae919efb9ec3451d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abbd8726472443eebb9365334eb634e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b0cbab21c872461084786fbea38a2094": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b836e939b1dc4762bec490893faac335": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd1fa2ec11d446aa9ad3952c8b35bf31": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cbd57b4ad48b4c8ca4b5616005fce6e7",
            "placeholder": "​",
            "style": "IPY_MODEL_3ee928be005c4d70b43b7f72d58672d9",
            "value": "tokenizer.json: 100%"
          }
        },
        "c423e62fa75e4b4189012c9ef12005a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9eb53e446d9f4b7dae919efb9ec3451d",
            "placeholder": "​",
            "style": "IPY_MODEL_b836e939b1dc4762bec490893faac335",
            "value": " 466k/466k [00:00&lt;00:00, 4.90MB/s]"
          }
        },
        "c6262084f781434fba9a273e0a606fff": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cbd57b4ad48b4c8ca4b5616005fce6e7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce74d6aa114144a196519687ef1a62fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cfab8cbfe5fa4f2087a73e697fd59b5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4eb5f692a9a14186bc0dafa12561073d",
              "IPY_MODEL_de321fe6c04c4b0eb2cf25d14f0b73d4",
              "IPY_MODEL_4d8c3670f82949c0997b29f7543b1482"
            ],
            "layout": "IPY_MODEL_1bcf03fa396b4761b9a29b8557124a69"
          }
        },
        "d3fd1b76eb6c48a9aa2c7216ac1032e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31e51667f892458db9264f68218c0954",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e79a0a56c6cd46c78fddeb532909be44",
            "value": 570
          }
        },
        "d7faf225275d4dea8adbf7ebd8adf7b9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9a83ce852d64639a8460152def0bef5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de321fe6c04c4b0eb2cf25d14f0b73d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6becf7082d3a4b0da24be1b9fae2ab2d",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14fef336c7204910a8608f51da1f6505",
            "value": 440449768
          }
        },
        "e79a0a56c6cd46c78fddeb532909be44": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ed8b133fed824ce9bed8a8f64384379d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_77e128871ed6448184b7bcd8a7a483db",
              "IPY_MODEL_53bfe556bfe841fd95a24e3563792591",
              "IPY_MODEL_4940afd924ad4d37b2c45ba64f75a5be"
            ],
            "layout": "IPY_MODEL_86a7ccd61ab545c8a2f5e63101155e79"
          }
        },
        "efcaafee43ea47848fed5aad4e2fd5bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_85999c408377499ba4a6bfee4ff42479",
              "IPY_MODEL_d3fd1b76eb6c48a9aa2c7216ac1032e2",
              "IPY_MODEL_780ba8f6d5c94c66b8f2b617cb387974"
            ],
            "layout": "IPY_MODEL_9d723a4d5d3a4c6db1d5bfa1ee5ea408"
          }
        },
        "f526f21fe01443399f044811b54a99c9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f642ce3da3b349439feb9fb9118e4c05": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd1c91fa5d074d8abb7dd15b7ef81c20": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0cbab21c872461084786fbea38a2094",
            "placeholder": "​",
            "style": "IPY_MODEL_abbd8726472443eebb9365334eb634e7",
            "value": "tokenizer_config.json: 100%"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}