Pytorch_Lightning_BERT_Huggingface.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pytorch Lightning BERT Huggingface.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e2a64f4e34684ad69880f9adc899cc34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6f6854375c4b4761947858f66ca6f6e1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_42e67889c55244529e39569037d38199",
              "IPY_MODEL_e02465515b0146378d4053bcd6fb0cbc",
              "IPY_MODEL_c1abd35d7890489fbff6f72871af97db"
            ]
          }
        },
        "6f6854375c4b4761947858f66ca6f6e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "42e67889c55244529e39569037d38199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a98b5eb71bd347138f20133850eb0d47",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Validation sanity check: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_49d4b5fb858e4810828e190fe10b3752"
          }
        },
        "e02465515b0146378d4053bcd6fb0cbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c4bd205d8ae24b198c59d6b595917b7d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e77bcc8ee1e4e2697e53d9a8b5c6b7b"
          }
        },
        "c1abd35d7890489fbff6f72871af97db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af7cc96b97f746e4944e6fb48e526886",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2/2 [00:00&lt;00:00,  2.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5796b93450dc429d858ffc4fd8809c39"
          }
        },
        "a98b5eb71bd347138f20133850eb0d47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "49d4b5fb858e4810828e190fe10b3752": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4bd205d8ae24b198c59d6b595917b7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e77bcc8ee1e4e2697e53d9a8b5c6b7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af7cc96b97f746e4944e6fb48e526886": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5796b93450dc429d858ffc4fd8809c39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4e0d20ec8d940c0adcf68d779373679": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2a2206d373dc477aa2148822e1dd98cb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_488cb9a9d7374f53a85e70e7e346afb0",
              "IPY_MODEL_e243a247b3344734a81d978b8c41f026",
              "IPY_MODEL_d701948b69ad49db9324f0bdadd0634e"
            ]
          }
        },
        "2a2206d373dc477aa2148822e1dd98cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "488cb9a9d7374f53a85e70e7e346afb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_50fbddc2c634477ab478d3085597579b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Epoch 1: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ed3a5671f8c4f189516bc108883dda5"
          }
        },
        "e243a247b3344734a81d978b8c41f026": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_97b4b9dd58a84fe0b3f472b7f68c643e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 84,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 84,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42c9d6b36036410eb7be839b0d78a0fa"
          }
        },
        "d701948b69ad49db9324f0bdadd0634e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_51cdf2ec5cc340e9ae9dda312c4293e1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 84/84 [00:38&lt;00:00,  2.20it/s, loss=0.17, v_num=5, loss_step=0.219, train_f1_step=0.762, val_loss_step=0.0453, val_f1_step=0.000, val_loss_epoch=0.094, val_f1_epoch=0.540, avg_val_loss=0.0936, avg_val_f1=0.536, loss_epoch=0.168, train_f1_epoch=0.787, avg_train_loss=0.042, avg_train_accuracy=0.787]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_688e227b23ad43d4b5527d83a7c1ea2c"
          }
        },
        "50fbddc2c634477ab478d3085597579b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ed3a5671f8c4f189516bc108883dda5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97b4b9dd58a84fe0b3f472b7f68c643e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42c9d6b36036410eb7be839b0d78a0fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51cdf2ec5cc340e9ae9dda312c4293e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "688e227b23ad43d4b5527d83a7c1ea2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37d8c657821142e581af349eaccc8929": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2eed335550f94202a6a5be5e64fa538f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d0f2bd267ccc47f8b2115524305a5209",
              "IPY_MODEL_ae0f650c8a634978965909b9c75efeb3",
              "IPY_MODEL_30af2edca8254acba212f1c699b15fe2"
            ]
          }
        },
        "2eed335550f94202a6a5be5e64fa538f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "d0f2bd267ccc47f8b2115524305a5209": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_08182aa96f044122b36a5086ec0a482d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Validating: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b956bae3d5064acd8d4494f7540f048f"
          }
        },
        "ae0f650c8a634978965909b9c75efeb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_51968e8c46fa4deb84d3e7414bd8d16f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 42,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 42,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3524659384774397818685816190fad2"
          }
        },
        "30af2edca8254acba212f1c699b15fe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4da6142797ae40dba0a6e21ece561bee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 42/42 [00:08&lt;00:00,  4.87it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c8a49a62a264994bf01cd9404c3aa05"
          }
        },
        "08182aa96f044122b36a5086ec0a482d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b956bae3d5064acd8d4494f7540f048f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51968e8c46fa4deb84d3e7414bd8d16f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3524659384774397818685816190fad2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4da6142797ae40dba0a6e21ece561bee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c8a49a62a264994bf01cd9404c3aa05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d54cc98a7434eac87c5c5a0760682b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0c56c526a18445b18b98940b7e2982de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_853be563d1d54dba8c72aa8564c894e2",
              "IPY_MODEL_ff37e1267eea4208afa8b88705b9caa4",
              "IPY_MODEL_c0a2105e0cc64d0b9f37dda8d5fcb3e4"
            ]
          }
        },
        "0c56c526a18445b18b98940b7e2982de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "853be563d1d54dba8c72aa8564c894e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d6cfd51638bb432e9d1b172a7f93f011",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Validating: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4296843d9e5448638115f2198630fba1"
          }
        },
        "ff37e1267eea4208afa8b88705b9caa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_335751ac5a5a4c7dbc0eb73be9e143a3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 42,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 42,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2748824093b4f73817ff5bf3f524dbf"
          }
        },
        "c0a2105e0cc64d0b9f37dda8d5fcb3e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f03ad422531a4a949b4b5b043a01b6cb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 42/42 [00:08&lt;00:00,  4.86it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8d5967f64ecb4ea09cdf911e4d61e1e3"
          }
        },
        "d6cfd51638bb432e9d1b172a7f93f011": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4296843d9e5448638115f2198630fba1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "335751ac5a5a4c7dbc0eb73be9e143a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2748824093b4f73817ff5bf3f524dbf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f03ad422531a4a949b4b5b043a01b6cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8d5967f64ecb4ea09cdf911e4d61e1e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/maxmatical/ml-cheatsheet/blob/master/Pytorch_Lightning_BERT_Huggingface.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nzBFZ3cJG7AV"
      },
      "source": [
        "# Inspirations\n",
        "https://curiousily.com/posts/multi-label-text-classification-with-bert-and-pytorch-lightning/\n",
        "\n",
        "https://github.com/mgrankin/over9000/blob/master/train.py\n",
        "\n",
        "\n",
        "https://medium.com/pytorch/getting-started-with-ray-lightning-easy-multi-node-pytorch-lightning-training-e639031aff8b\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1o9pMmzzoq7e",
        "outputId": "023f86d8-5d23-49fd-e247-06b9ba5a9e95"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wed Dec 29 20:36:15 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 495.44       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   66C    P0    88W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QkBmYey_BMu"
      },
      "source": [
        "%%capture\n",
        "!pip install transformers\n",
        "!pip install pytorch_lightning\n",
        "!pip install torchmetrics"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zDygGNnC_VGv"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torch.optim.lr_scheduler import OneCycleLR # , ReduceLROnPlateau\n",
        "from torch.optim import AdamW\n",
        "\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModel\n",
        "  \n",
        "import pytorch_lightning as pl\n",
        "from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor, StochasticWeightAveraging\n",
        "from pytorch_lightning.loggers import TensorBoardLogger\n",
        "import torchmetrics\n",
        "from torchmetrics.functional import accuracy, f1, auroc\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report, multilabel_confusion_matrix"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SWso8EH2IDcU",
        "outputId": "f3b86bad-0bd0-48bd-c89c-f6eb0777dae6"
      },
      "source": [
        "# pretrained model and tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilroberta-base\")\n",
        "model = AutoModel.from_pretrained(\"distilroberta-base\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at distilroberta-base were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.bias']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4lqOT-HXGYIN"
      },
      "source": [
        "# Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7WLrlT3NGZGT",
        "outputId": "1cee9ac1-d971-451c-dac1-3b5e0c117d6e"
      },
      "source": [
        "!gdown --id 1VuQ-U7TtggShMeuRSA_hzC8qGDl2LRkr"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1VuQ-U7TtggShMeuRSA_hzC8qGDl2LRkr\n",
            "To: /content/toxic_comments.csv\n",
            "100% 68.8M/68.8M [00:00<00:00, 119MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "xuywc9Y5Gbf1",
        "outputId": "f8c9d97e-ef51-41f4-fa13-a63886285e67"
      },
      "source": [
        "df = pd.read_csv(\"toxic_comments.csv\")\n",
        "\n",
        "df.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-135f19a0-50a3-425e-815d-9e5e0da7e454\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-135f19a0-50a3-425e-815d-9e5e0da7e454')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-135f19a0-50a3-425e-815d-9e5e0da7e454 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-135f19a0-50a3-425e-815d-9e5e0da7e454');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                 id  ... identity_hate\n",
              "0  0000997932d777bf  ...             0\n",
              "1  000103f0d9cfb60f  ...             0\n",
              "2  000113f07ec002fd  ...             0\n",
              "3  0001b41b1c6bb37e  ...             0\n",
              "4  0001d958c54c6e35  ...             0\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bDVf_wTaGgVP"
      },
      "source": [
        "train_df, val_df = train_test_split(df, test_size=0.15)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ro0J6WPGv69",
        "outputId": "27f9702e-2a61-4204-d959-b3c6075276af"
      },
      "source": [
        "# subsample clean comments\n",
        "LABEL_COLUMNS = df.columns.tolist()[2:]\n",
        "\n",
        "train_toxic = train_df[train_df[LABEL_COLUMNS].sum(axis=1) > 0]\n",
        "train_clean = train_df[train_df[LABEL_COLUMNS].sum(axis=1) == 0]\n",
        "\n",
        "train_df = pd.concat([\n",
        "  train_toxic,\n",
        "  train_clean.sample(15_000)\n",
        "])\n",
        "\n",
        "train_df.shape, val_df.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((28787, 8), (23936, 8))"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EF3tWEZ74Vww",
        "outputId": "d03c6da8-a74a-4e9a-9bff-751ad7dcfb1d"
      },
      "source": [
        "# take only a subsample of each train_df and val_df for faster iterations\n",
        "train_df = train_df.sample(500)\n",
        "val_df = val_df.sample(500)\n",
        "\n",
        "train_df.shape, val_df.shape"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 8), (500, 8))"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dn-Q0BjUoB9Y"
      },
      "source": [
        "## Creating Dataset and Lightning Data Module"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QV-deDgoo1Rx"
      },
      "source": [
        "# set batch size and max seq_len\n",
        "bs = 12\n",
        "seq_len = 256"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RL6vWIO4HoO2"
      },
      "source": [
        "class ToxicCommentsDataset(Dataset):\n",
        "\n",
        "  def __init__(\n",
        "    self,\n",
        "    data: pd.DataFrame,\n",
        "    tokenizer: AutoTokenizer,\n",
        "    max_token_len: int = 128\n",
        "\n",
        "  ):\n",
        "    self.tokenizer = tokenizer\n",
        "    self.data = data\n",
        "    self.max_token_len = max_token_len\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.data)\n",
        "\n",
        "  def __getitem__(self, index: int):\n",
        "    data_row = self.data.iloc[index]\n",
        "    comment_text = data_row.comment_text\n",
        "    labels = data_row[LABEL_COLUMNS]\n",
        "\n",
        "    encoding = self.tokenizer.encode_plus(\n",
        "      comment_text,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_token_len,\n",
        "      return_token_type_ids=False,\n",
        "      padding=\"max_length\",\n",
        "      truncation=True,\n",
        "      return_attention_mask=True,\n",
        "      return_tensors='pt',\n",
        "\n",
        "    )\n",
        "\n",
        "    return dict(\n",
        "      # comment_text=comment_text,\n",
        "      input_ids=encoding[\"input_ids\"].flatten(),\n",
        "      attention_mask=encoding[\"attention_mask\"].flatten(),\n",
        "      labels = torch.IntTensor(labels)\n",
        "    #   labels=torch.FloatTensor(labels)\n",
        "    )"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mMPUJ8inH2Jg",
        "outputId": "822133c8-7104-4b39-c091-a6dc19f60c05"
      },
      "source": [
        "# test\n",
        "train_dataset = ToxicCommentsDataset(\n",
        "  train_df,\n",
        "  tokenizer,\n",
        "  max_token_len=seq_len\n",
        ")\n",
        "\n",
        "sample_item = train_dataset[0]\n",
        "sample_item.keys()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask', 'labels'])"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vvh5OltLILMa",
        "outputId": "264498da-f833-4eba-e243-20c20a3810cc"
      },
      "source": [
        "print(sample_item[\"input_ids\"], sample_item[\"labels\"])\n",
        "print(sample_item[\"input_ids\"].shape)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([    0,  1185, 11216,  1368,  3540,     4,  1437, 50118, 50118, 25266,\n",
            "         1810,     6, 32594,     4,     2,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
            "            1,     1,     1,     1,     1,     1]) tensor([1, 0, 1, 0, 1, 0], dtype=torch.int32)\n",
            "torch.Size([256])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0gIaph4IkSu"
      },
      "source": [
        "class ToxicCommentsDataModule(pl.LightningDataModule):\n",
        "  def __init__(self, train_df, test_df, tokenizer, batch_size=8, max_token_len=512):\n",
        "    super().__init__()\n",
        "    self.train_df, self.test_df = train_df, test_df\n",
        "    self.tokenizer = tokenizer\n",
        "    self.batch_size = batch_size\n",
        "    self.max_token_len = max_token_len\n",
        "\n",
        "  def setup(self, stage=None):\n",
        "    self.train_dataset = ToxicCommentsDataset(\n",
        "        self.train_df,\n",
        "        self.tokenizer,\n",
        "        self.max_token_len\n",
        "    )\n",
        "\n",
        "    self.test_dataset = ToxicCommentsDataset(\n",
        "        self.test_df,\n",
        "        self.tokenizer,\n",
        "        self.max_token_len\n",
        "    )\n",
        "\n",
        "  def train_dataloader(self):\n",
        "    return DataLoader(\n",
        "        self.train_dataset,\n",
        "        batch_size = self.batch_size,\n",
        "        shuffle = True,\n",
        "        num_workers=1\n",
        "    )\n",
        "\n",
        "  def val_dataloader(self):\n",
        "    return DataLoader(\n",
        "        self.test_dataset,\n",
        "        batch_size = self.batch_size,\n",
        "        shuffle = False,\n",
        "        num_workers=1\n",
        "    )\n",
        "\n",
        "  def test_dataloader(self):\n",
        "    return DataLoader(\n",
        "        self.test_dataset,\n",
        "        batch_size = self.batch_size,\n",
        "        shuffle = False,\n",
        "        num_workers=1\n",
        "    )"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwxqBy-FIjf5"
      },
      "source": [
        "data_module = ToxicCommentsDataModule(\n",
        "    train_df,\n",
        "    val_df,\n",
        "    tokenizer,\n",
        "    batch_size = bs,\n",
        "    max_token_len = seq_len\n",
        ")\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8zCGQwkFqZD8"
      },
      "source": [
        "data_module.setup() # call this before getting len of dataloader"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iM2A5D6tqteX",
        "outputId": "fb88cd71-4c2e-4ec9-c5ee-38839aeffc82"
      },
      "source": [
        "len(data_module.train_dataloader())"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "42"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D1zQOd2AJl7l"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a99EheTsnJyK",
        "outputId": "c9f8bd91-c9f7-465f-d33f-a9f1dee7019b"
      },
      "source": [
        "sample_batch = next(iter(DataLoader(train_dataset, batch_size=8, num_workers=1)))\n",
        "sample_batch[\"input_ids\"].shape, sample_batch[\"attention_mask\"].shape"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([8, 256]), torch.Size([8, 256]))"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkQzJyrnnPyc"
      },
      "source": [
        "output = model(sample_batch[\"input_ids\"], sample_batch[\"attention_mask\"])"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gTsZW4_anUYU",
        "outputId": "24922b02-fe4b-4b03-d78f-9a2123817794"
      },
      "source": [
        "output.pooler_output.shape"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([8, 768])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8LOwh55Okdz"
      },
      "source": [
        "# test lr schedule with https://www.kaggle.com/isbhargav/guide-to-pytorch-learning-rate-scheduling\n",
        "# test wtih flat cos lr"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ad0Y-3mUXmRM"
      },
      "source": [
        "class BertModel(nn.Module):\n",
        "  def __init__(self, n_classes: int):\n",
        "    super().__init__()\n",
        "    self.model = model\n",
        "    self.classifier = nn.Linear(self.model.config.hidden_size, n_classes)\n",
        "\n",
        "  def forward(self, input_ids, attention_mask): \n",
        "    out = self.model(input_ids, attention_mask=attention_mask)\n",
        "    out = self.classifier(out.pooler_output)\n",
        "    return out"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Eb5FtVEYWQQ"
      },
      "source": [
        "bert_model = BertModel(len(LABEL_COLUMNS))"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uc6_SAgKYmmC"
      },
      "source": [
        "# bert_model(sample_batch[\"input_ids\"], sample_batch[\"attention_mask\"]).shape  # should be bs x 6"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "configuring multiple optimizers: https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html?highlight=scheduler#configure-optimizers"
      ],
      "metadata": {
        "id": "nF_h1Iyk2WYt"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XgaReoIsJh0M"
      },
      "source": [
        "class ToxicCommentClassifier(pl.LightningModule):\n",
        "  def __init__(self, pytorch_model: nn.Module, total_steps: int, lr: float = 2e-5, accumulate_grad_batches: int = 1):\n",
        "    super().__init__()\n",
        "    self.lr = lr\n",
        "    self.total_steps = total_steps # for lr schedule\n",
        "    self.accumulate_grad_batches = accumulate_grad_batches # used as step freq for lr schedule\n",
        "\n",
        "    self.pytorch_model = pytorch_model\n",
        "    self.criterion = nn.BCEWithLogitsLoss() # bce for multi-label\n",
        "    # self.criterion = nn.BCELoss()\n",
        "\n",
        "    # metrics \n",
        "    self.f1 = torchmetrics.F1()\n",
        "\n",
        "    # use automatic optimization to true\n",
        "    self.automatic_optimization = True\n",
        "\n",
        "    self.lrs = []\n",
        "\n",
        "  def forward(self, input_ids, attention_mask):\n",
        "    \"\"\"\n",
        "    forward step for lightning module\n",
        "    \"\"\"\n",
        "    out = self.pytorch_model(input_ids, attention_mask=attention_mask)\n",
        "    return out\n",
        "\n",
        "  def training_step(self, batch, batch_idx):\n",
        "    \"\"\"\n",
        "    using automatic optimization: no need to define opt steps etc.\n",
        "    \"\"\"\n",
        "    input_ids = batch[\"input_ids\"]\n",
        "    attention_mask = batch[\"attention_mask\"]\n",
        "    labels = batch[\"labels\"]\n",
        "\n",
        "    out = self(input_ids, attention_mask)\n",
        "       \n",
        "    # compute loss + metrics\n",
        "    loss = self.criterion(out, labels.to(dtype=torch.float32)) # cast to float because labels is in Int\n",
        "    f1 = self.f1(out, labels)\n",
        "\n",
        "    # log loss and metrics each k steps and each epoch\n",
        "    log_values = {\"loss\": loss, \"train_f1\": f1}\n",
        "    self.log_dict(log_values, sync_dist=True, prog_bar=True, on_step=True, on_epoch=True)\n",
        "\n",
        "    # track lr just to see if things are working correctly\n",
        "    curr_lr = self.scheduler.get_last_lr()\n",
        "    self.lrs.append(curr_lr[0])\n",
        "\n",
        "\n",
        "    return {\"loss\": loss, \"train_f1\": f1}\n",
        "\n",
        "  def validation_step(self, batch, batch_idx):\n",
        "    \"\"\"\n",
        "    validation step\n",
        "    get validation loss and accuracy metrics\n",
        "    \"\"\"\n",
        "    # batch comes from dataset\n",
        "    input_ids = batch[\"input_ids\"]\n",
        "    attention_mask = batch[\"attention_mask\"]\n",
        "    labels = batch[\"labels\"]\n",
        "\n",
        "    out = self(input_ids, attention_mask)\n",
        "\n",
        "    # compute loss + metrics\n",
        "    loss = self.criterion(out, labels.to(dtype=torch.float32)) # cast to float because labels is in Int\n",
        "    f1 = self.f1(out, labels)\n",
        "    # log loss and metrics each k steps and each epoch\n",
        "    log_values = {\"val_loss\": loss, \"val_f1\": f1}\n",
        "    self.log_dict(log_values, sync_dist=True, prog_bar=True, on_step=True, on_epoch=True)\n",
        "    return {\"val_loss\": loss, \"val_f1\": f1}\n",
        "\n",
        "  def training_epoch_end(self, outputs):\n",
        "    \"\"\"\n",
        "    after every training epoch, log train metrics\n",
        "\n",
        "    note: this might not be needed if logging is done for each step and epoch\n",
        "    \"\"\"\n",
        "    # log validation loss and metrics\n",
        "    avg_loss = torch.stack([x[\"loss\"] for x in outputs]).mean()\n",
        "    avg_f1 = torch.stack([x[\"train_f1\"] for x in outputs]).mean()\n",
        "\n",
        "    self.log(\"avg_train_loss\", avg_loss, sync_dist=True, prog_bar=True)\n",
        "    self.log(\"avg_train_f1\", avg_f1, sync_dist=True, prog_bar=True)\n",
        "\n",
        "  def validation_epoch_end(self, outputs):\n",
        "    \"\"\"\n",
        "    after every training epoch, log val metrics\n",
        "\n",
        "    note: this might not be needed if logging is done for each step and epoch\n",
        "    \"\"\"\n",
        "    # log validation loss and metrics\n",
        "    avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
        "    avg_f1 = torch.stack([x[\"val_f1\"] for x in outputs]).mean()\n",
        "    self.log(\"avg_val_loss\", avg_loss, sync_dist=True, prog_bar=True)\n",
        "    self.log(\"avg_val_f1\", avg_f1, sync_dist=True, prog_bar=True)\n",
        "\n",
        "  def configure_optimizers(self):\n",
        "    # TODO: \n",
        "    # 1. add flat cos\n",
        "    # 2. make lr scheduler configurable(one_cycle vs flat_cos)\n",
        "    # 3. potentially add reduce lr on plateau as well\n",
        "\n",
        "    optimizer = AdamW(self.parameters(), lr=self.lr)\n",
        "\n",
        "    self.scheduler = OneCycleLR(\n",
        "      optimizer,\n",
        "      max_lr=self.lr,\n",
        "      pct_start=0.3,\n",
        "      total_steps=self.total_steps\n",
        "    )\n",
        "    scheduler_dic = {\n",
        "        'scheduler': self.scheduler,\n",
        "        'interval': 'step',\n",
        "        \"frequency\": self.accumulate_grad_batches\n",
        "    }\n",
        "    return [optimizer], [scheduler_dic]"
      ],
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHhFIJkGUTU4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b63b01d-748b-4abb-d12c-b13f6cc1da1e"
      },
      "source": [
        "# training epoch related hyperparams\n",
        "steps_per_epoch = len(data_module.train_dataloader())\n",
        "n_epochs = 2\n",
        "accumulate_grad_batches = 4\n",
        "\n",
        "total_steps = steps_per_epoch * n_epochs // accumulate_grad_batches\n",
        "print(f\"total_steps: {total_steps}\")"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total_steps: 21\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPhpGLWyV0eL"
      },
      "source": [
        "Note: if using multiple gpus, total steps calculation is something like\n",
        "```\n",
        "steps_per_epoch = len(data_module.train_dataloader())\n",
        "\n",
        "# for gpu or tpu cores\n",
        "num_devices = 4\n",
        "if tpu_cores:\n",
        "    num_devices = max(num_devices, tpu_cores)\n",
        "\n",
        "accumulate_grad_batches = 1 # for no gradient accumulation\n",
        "\n",
        "effective_accum = accumulate_grad_batches * num_devices\n",
        "total_steps (steps_per_epoch // effective_accum) * n_epochs\n",
        "\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O5o32pPDKTTi"
      },
      "source": [
        "toxic_comment_model = ToxicCommentClassifier(bert_model, lr=2e-5, total_steps=total_steps, accumulate_grad_batches=accumulate_grad_batches)"
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZR27IbG2Y8o2"
      },
      "source": [
        "# toxic_comment_model(sample_batch[\"input_ids\"], sample_batch[\"attention_mask\"])[1].shape  # should be bs x 6"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "li0_cLa82LXx"
      },
      "source": [
        "# Trainer\n",
        "\n",
        "defining the trainer and running `trainer.fit` "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrBKqeKnKWy-"
      },
      "source": [
        "# callbacks\n",
        "lr_monitor_cb = LearningRateMonitor(logging_interval='step')\n",
        "\n",
        "checkpoint_callback = ModelCheckpoint(\n",
        "  dirpath=\"checkpoints\",\n",
        "  filename=\"best-checkpoint\",\n",
        "  save_top_k=1,\n",
        "  verbose=True,\n",
        "  monitor=\"avg_val_f1\",\n",
        "  mode=\"max\"\n",
        ")\n",
        "\n",
        "early_stopping_callback = EarlyStopping(monitor='val_loss', patience=6)\n",
        "\n",
        "# swa = StochasticWeightAveraging(swa_epoch_start=0.8, swa_lrs=None)\n",
        "\n",
        "logger = TensorBoardLogger(\"lightning_logs\", name=\"toxic-comments\")"
      ],
      "execution_count": 84,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RCkRvDZqzx6"
      },
      "source": [
        "# standard callback, checkpoint and early stopping\n",
        "callbacks = [lr_monitor_cb, checkpoint_callback, early_stopping_callback]"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cZdITHvZhBSF",
        "outputId": "bba4c842-ccd4-444f-899c-46a29eff6ed7"
      },
      "source": [
        "trainer = pl.Trainer(\n",
        "    logger=logger,\n",
        "    callbacks = callbacks,\n",
        "    precision=32,\n",
        "    max_epochs = n_epochs,\n",
        "    accelerator=\"gpu\",\n",
        "    devices=1,\n",
        "    accumulate_grad_batches = accumulate_grad_batches\n",
        ")"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "IPU available: False, using: 0 IPUs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSIidqFNT5n5"
      },
      "source": [
        "# optional: lr finder\n",
        "# lr_finder = trainer.tuner.lr_find(model=toxic_comment_model, datamodule=data_module)"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 533,
          "referenced_widgets": [
            "e2a64f4e34684ad69880f9adc899cc34",
            "6f6854375c4b4761947858f66ca6f6e1",
            "42e67889c55244529e39569037d38199",
            "e02465515b0146378d4053bcd6fb0cbc",
            "c1abd35d7890489fbff6f72871af97db",
            "a98b5eb71bd347138f20133850eb0d47",
            "49d4b5fb858e4810828e190fe10b3752",
            "c4bd205d8ae24b198c59d6b595917b7d",
            "8e77bcc8ee1e4e2697e53d9a8b5c6b7b",
            "af7cc96b97f746e4944e6fb48e526886",
            "5796b93450dc429d858ffc4fd8809c39",
            "b4e0d20ec8d940c0adcf68d779373679",
            "2a2206d373dc477aa2148822e1dd98cb",
            "488cb9a9d7374f53a85e70e7e346afb0",
            "e243a247b3344734a81d978b8c41f026",
            "d701948b69ad49db9324f0bdadd0634e",
            "50fbddc2c634477ab478d3085597579b",
            "3ed3a5671f8c4f189516bc108883dda5",
            "97b4b9dd58a84fe0b3f472b7f68c643e",
            "42c9d6b36036410eb7be839b0d78a0fa",
            "51cdf2ec5cc340e9ae9dda312c4293e1",
            "688e227b23ad43d4b5527d83a7c1ea2c",
            "37d8c657821142e581af349eaccc8929",
            "2eed335550f94202a6a5be5e64fa538f",
            "d0f2bd267ccc47f8b2115524305a5209",
            "ae0f650c8a634978965909b9c75efeb3",
            "30af2edca8254acba212f1c699b15fe2",
            "08182aa96f044122b36a5086ec0a482d",
            "b956bae3d5064acd8d4494f7540f048f",
            "51968e8c46fa4deb84d3e7414bd8d16f",
            "3524659384774397818685816190fad2",
            "4da6142797ae40dba0a6e21ece561bee",
            "9c8a49a62a264994bf01cd9404c3aa05",
            "5d54cc98a7434eac87c5c5a0760682b8",
            "0c56c526a18445b18b98940b7e2982de",
            "853be563d1d54dba8c72aa8564c894e2",
            "ff37e1267eea4208afa8b88705b9caa4",
            "c0a2105e0cc64d0b9f37dda8d5fcb3e4",
            "d6cfd51638bb432e9d1b172a7f93f011",
            "4296843d9e5448638115f2198630fba1",
            "335751ac5a5a4c7dbc0eb73be9e143a3",
            "d2748824093b4f73817ff5bf3f524dbf",
            "f03ad422531a4a949b4b5b043a01b6cb",
            "8d5967f64ecb4ea09cdf911e4d61e1e3"
          ]
        },
        "id": "Jfl3uth_hSRg",
        "outputId": "19bbfb8f-17d3-43f7-bbec-3ab8e851b1f6"
      },
      "source": [
        "trainer.fit(toxic_comment_model, data_module)"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/datamodule.py:470: LightningDeprecationWarning: DataModule.setup has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.setup.\n",
            "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name          | Type              | Params\n",
            "----------------------------------------------------\n",
            "0 | pytorch_model | BertModel         | 82.1 M\n",
            "1 | criterion     | BCEWithLogitsLoss | 0     \n",
            "2 | f1            | F1                | 0     \n",
            "----------------------------------------------------\n",
            "82.1 M    Trainable params\n",
            "0         Non-trainable params\n",
            "82.1 M    Total params\n",
            "328.492   Total estimated model params size (MB)\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/callbacks/model_checkpoint.py:631: UserWarning: Checkpoint directory /content/checkpoints exists and is not empty.\n",
            "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e2a64f4e34684ad69880f9adc899cc34",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Validation sanity check: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/data_loading.py:413: UserWarning: The number of training samples (42) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
            "  f\"The number of training samples ({self.num_training_batches}) is smaller than the logging interval\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b4e0d20ec8d940c0adcf68d779373679",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Training: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "37d8c657821142e581af349eaccc8929",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Validating: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 0, global step 10: avg_val_f1 reached 0.50271 (best 0.50271), saving model to \"/content/checkpoints/best-checkpoint-v3.ckpt\" as top 1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5d54cc98a7434eac87c5c5a0760682b8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Validating: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1, global step 21: avg_val_f1 reached 0.53552 (best 0.53552), saving model to \"/content/checkpoints/best-checkpoint-v3.ckpt\" as top 1\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/datamodule.py:470: LightningDeprecationWarning: DataModule.teardown has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.teardown.\n",
            "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# check lr scheduler working as intended\n",
        "len(toxic_comment_model.lrs), max(toxic_comment_model.lrs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mS3oSTQR6-4v",
        "outputId": "aca0c5e4-e3ce-4c17-91a1-379dffb1d4b9"
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(84, 1.9888308709018238e-05)"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib import pyplot as plt\n",
        "\n",
        "plt.plot(toxic_comment_model.lrs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "id": "fbNMk46391UV",
        "outputId": "78eb9d93-503e-4572-ae4b-68a947a10cdd"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fd58f9d0cd0>]"
            ]
          },
          "metadata": {},
          "execution_count": 91
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEDCAYAAAA7jc+ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXic5Xnv8e+t3Ys2W7IkL/ICsmUb8ILCvq8mSaFtcgK0aWlL6tITSrYuoUvgkJyWtClt2pAEh7gkbQNJgKRO2OJTQk3CEmSbxRvg3fKCbUm2bMna7/PHvDJjI1mDZkbvLL/Pdemy5l1Gt8ajnx497/M+j7k7IiKSuXLCLkBERJJLQS8ikuEU9CIiGU5BLyKS4RT0IiIZTkEvIpLhUjbozWy5me03s3UJer4+M3s1+FiRiOcUEUkHlqrj6M3sEuAo8F13PyMBz3fU3cfHX5mISHpJ2Ra9u68CWqK3mdlpZva0ma02s+fNrD6k8kRE0kbKBv0QlgF/4u5nA38KfP19nFtkZo1m9pKZ/XpyyhMRST15YRcQKzMbD1wA/NDMBjYXBvt+E7hnkNN2u/u1wefT3X23mc0CnjWzN9x9S7LrFhEJW9oEPZG/Pg65+8KTd7j748DjpzrZ3XcH/241s+eARYCCXkQyXtp03bh7G7DNzP4XgEUsiOVcMys3s4HWfwVwIbAhacWKiKSQlA16M3sYeBGYY2ZNZnYr8NvArWb2GrAeuCHGp5sLNAbn/Ry4190V9CKSFVJ2eKWIiCRGyrboRUQkMVLyYmxFRYXPmDEj7DJERNLG6tWrD7p75WD7UjLoZ8yYQWNjY9hliIikDTPbMdQ+dd2IiGQ4Bb2ISIZT0IuIZDgFvYhIhhs26M1smpn93Mw2mNl6M/vUIMeYmf2LmW02s9fNbHHUvlvM7O3g45ZEfwMiInJqsYy66QU+5+5rzKwYWG1mK0+6s/Q6oC74OBf4BnCumU0A7gIaAA/OXeHurQn9LkREZEjDtujdfa+7rwk+PwJsBKacdNgNRBYIcXd/CSgzsxrgWmClu7cE4b4SWJLQ70BERE7pfY2jN7MZRGZ9fPmkXVOAXVGPm4JtQ20f7LmXAksBamtr309Z8j709zv72jpjPr6ju5d9h7vYe/gY77R10t3bH9N5ZsZvLp7C9InjRlqqiCRIzEEfzAf/GPDpYCbJhHL3ZUQWFqGhoUET8CTJ3z21kW89v23E57+7FMCpucNPXtvDE3dczJiC3BF/PRGJX0xBb2b5REL+P4O530+2G5gW9XhqsG03cNlJ258bSaGSGK81HWZW5Tj+6JJZMR1flJ9LdUkR1aVFVJUUUZQfW2i/sPkgv/Xgy3z56U3cff38eEoWkTgNG/QWWc7p28BGd79viMNWALeb2SNELsYedve9ZvYM8LdmVh4cdw1wZwLqlhHafrCdi+squfEDye0eu+D0Cn7vghk89MJ2rplXxQWnVyT164nI0GIZR38h8DvAFWb2avDxQTO7zcxuC455EtgKbAa+BfxvAHdvAb4IvBJ83BNskxB0dPey/0gXMyvGjsrX+4sl9cyqGMefPfo6Rzp7RuVrish7Dduid/dfAKfsmfXIpPafHGLfcmD5iKqThNp+sAOAGRWjc4F0TEEuX/nYAj76jRe4e8UGPn1VXUznFeXnUllcmOTqRLJHSs5eKcmxvbkdgBmjOBJmcW05t116Gl9/bguPrWmK+bwff/JCFk4rS2JlItlDQZ9Fth0Mgn6UWvQDPnv1bBZMK+NIZ++wx7o7d61Yz8Mv71TQiySIgj6LbD/YTmVxIeMLR/e/PS83h2vnV8d8/K+2tfDT1/dw1/XzGFugt6hIvDSpWRbZ3tzOjImjcyE2Hh/7wDTau/t48o19YZcikhEU9Flk28GOUe2fH6mG6eXMrBjHDxp3DX+wiAxLQZ8ljnT2cPBo16j3z4+EmfHRs6fyq20tx68riMjIKeizxI7myNDKmWkQ9AAfWTyVHINHV6tVLxIvBX2WCGNoZTyqS4u4dHYlj63eTV+/pj4SiYeCPktsPz60MvUvxg74WMM09rV18vzbB8IuRSStaexalth2sIOqksK0Gq545dwqysfmc9/Kt/jVtthmzjhzSinXnVmT5MpE0kv6/NRLXCJDK9Oj22ZAQV4Ov3/hTP712bfZuHf4mbH7+p383Bwur58U8yybItlAQZ8lth9s56q5VWGX8b7dcWUdd1wZ2xw5z256hz94qJHG7a1cVKfZMkUGqI8+C7R19tDc3p0WQyvjce7MieTnmvr0RU6ioM8CAxdiR2t64rCMK8zj7OnlrHr7YNiliKQUBX0WCGsyszBcXFfJxr1t7D8S+7q4IplOQZ8FBuahnz4h84P+krpKAH65Wa16kQHDBr2ZLTez/Wa2boj9fxa18tQ6M+szswnBvu1m9kawrzHRxUtsdjS3U1NalBWLdM+fXMKEcQU8/5aCXmRALC36h4AlQ+10939w94XuvpDIerD/c9JygZcH+xviK1VGalsaDq0cqZwc46LTK1j19kEiC5+JyLBB7+6rgFjXeb0ZeDiuiiThth9sz4r++QEX11Vw8GgXm/YdCbsUkZSQsD56MxtLpOX/WNRmB35mZqvNbOkw5y81s0YzazxwQMPjEuVwRw+tHT0ZP+Im2sVBP72GWYpEJPKGqV8DfnlSt81F7r7bzCYBK81sU/AXwnu4+zJgGUBDQ4P+5j6F1vZuvvTERjp7+oY99khXZPm+6VnSdQORCdFmV43n+bcPsvSS08IuRyR0iQz6mzip28bddwf/7jezHwHnAIMGvcTu5W3NPLamidoJYynIG/6PsgXTyjh7evkoVJY6Lq6r5N9f2kFnT5+mQ5Csl5CgN7NS4FLg41HbxgE57n4k+Pwa4J5EfL1s19rRA8D3/+g8akrHhFxNarq4roJv/2IbT6/bF/MvuSllY8jJsSRXJjL6hg16M3sYuAyoMLMm4C4gH8Ddvxkc9hvAz9w9ejmgKuBHZjbwdb7n7k8nrvTs1drRDUD52IKQK0ld586cSGFeDp/+/qsxn3PHFafz2WvmJLEqkXAMG/TufnMMxzxEZBhm9LatwIKRFiZDa23vpig/R10SpzCmIJfv/eF5MS9FuGzVFn65pZnPJrkukTBo9so01NrRwwS15od19vTymLtt3tzXxnde3EF3b39M1z1E0one0WnoUEc3ZQr6hFpcW053bz8bYpj3XiTdKOjTUGtHD+Xj8sMuI6Msqo20/NfsaA25EpHEU9CnodZ2tegTrbq0iMmlRazddSjsUkQSTkGfhlo7uikfqxZ9oi2aXq4WvWQkBX2a6e93Dh/TxdhkWDStjN2HjrG/TXPZS2ZR0KeZts4e+h113STB4mCEzpqd6r6RzKKgTzMt7cHNUroYm3DzJ5dQkJvD2p3qvpHMoqBPMwPTH6hFn3iFebnMn1LCGgW9ZBgFfZo5FEx/oD765FhcW87rTYfp7u0PuxSRhFHQp5mBFr3muUmORbVldPX2s2mfbpySzKGgTzOtQR99mfrok2KxbpySDKSgTzOtHd3k5RjFhZqmKBlqSouoKinUyBvJKAr6NNPa0UPZ2HyC6Z8lwcyMxbXlrN2lFr1kDjUL08yhjm71zyfZ4tpynlq3j7tXrCc3zoVIDPjYB6Yxu6o4McWJjICCPs20tCvok+2yOZU8sGorj65uivu52rt7aWnv5r4bFyagMpGRiWWFqeXAh4H97n7GIPsvA/4L2BZsetzd7wn2LQG+CuQCD7r7vQmqO2sd6uhh+sSxYZeR0eqqimn866sS8lx/8NArrN+jETwSrlj66B8ClgxzzPPuvjD4GAj5XOB+4DpgHnCzmc2Lp1gZmNBMLfp0MX9yCZsPHKWzpy/sUiSLDRv07r4KaBnBc58DbHb3re7eDTwC3DCC55GAu3Ooo4fycQr6dDF/cgl9/c6mfUfCLkWyWKJG3ZxvZq+Z2VNmNj/YNgXYFXVMU7BtUGa21MwazazxwIEDCSors7R399Hd168pitPI/MmlAKzfczjkSiSbJSLo1wDT3X0B8K/Aj0fyJO6+zN0b3L2hsrIyAWVlnoGbpdR1kz6mlo+hdEw+63arn17CE3fQu3ubux8NPn8SyDezCmA3MC3q0KnBNhmhQ8cnNFOLPl2YGfNqStigFr2EKO6gN7NqC+7eMbNzgudsBl4B6sxsppkVADcBK+L9etmstWNgimK16NPJGVNK2LjvCD19mihNwhHL8MqHgcuACjNrAu4C8gHc/ZvAR4E/NrNe4Bhwk7s70GtmtwPPEBleudzd1yflu8gSx4NeXTdpZf7kUrp7+9ly4Cj11SVhlyNZaNigd/ebh9n/NeBrQ+x7EnhyZKXJyd7to1fXTTqZPzkS7ut2tynoJRSa6yaNDExRXDpGQZ9OZlWOpyg/RyNvJDQK+jRyqKObkqI88nL135ZOcnOMuTUlukNWQqPESCMtHT1M0IXYtHTG5FI27Gmjv9/DLkWykII+jRzq6NZasWlq/uQSjnb1srOlI+xSJAsp6NNIZJ4b9c+no4E7ZNepn15CoKBPI63tPRpamaZmV48nL8fUTy+hUNCnkVZ13aStwrxc6qqKFfQSCgV9mujq7aOju48JWhQ8bc2fXML63YeJ3E8oMnq0wlSaeHeeG7Xo09UZk0t4dHUTd61YT0EMQ2THFubxyctPozAvdxSqk0ymoE8Tmv4g/V1UV0HF+AIei2GJwn6HYz19LJxWyhX1VaNQnWQyBX2aaNH0B2nv9EnFNP711TEd29bZw1l3/4xN+44o6CVu6qNPEwNdN5q5MjuUFOUzpWwMm/ZqZSqJn4I+TajrJvvMrSnmTS1BKAmgoE8TWnQk+8ypLmbLgaN09WphcYmPgj5NtLZ3MyY/l6J8jcDIFvXVJfT2O1v2t4ddiqQ5BX2aaOno1oRmWWZuTTEAm/bpJiuJz7BBb2bLzWy/ma0bYv9vm9nrZvaGmb1gZgui9m0Ptr9qZo2JLDzbHOroUbdNlpkxcRwFeTnqp5e4xdKifwhYcor924BL3f1M4IvAspP2X+7uC929YWQlCgxMaKYWfTbJy82hbtJ4NiroJU7DBr27rwJaTrH/BXdvDR6+BExNUG0SRS367DSnuphNe9V1I/FJ9A1TtwJPRT124Gdm5sAD7n5ya/84M1sKLAWora1NcFmp6fbvrWHd7timrd3Z0sFFp1ckuSJJNXOrS3h8zW5a2nWNRkYuYUFvZpcTCfqLojZf5O67zWwSsNLMNgV/IbxH8EtgGUBDQ0PGz/rU3+888cZe5lQVM6e6eNjjF9WW89Gz9cdStqmPuiB7wWn6RS8jk5CgN7OzgAeB69y9eWC7u+8O/t1vZj8CzgEGDfpsc6SrF3f46NlT+cTFs8IuR1LUQCNg094jCnoZsbiHV5pZLfA48Dvu/lbU9nFmVjzwOXANMOjInWzUdixyA1TJGPW7y9AqxxcycVyBRt5IXIZt0ZvZw8BlQIWZNQF3AfkA7v5N4AvARODrZgbQG4ywqQJ+FGzLA77n7k8n4XtIS4eDoC9V0MspmBn1NcUaSy9xGTbo3f3mYfZ/AvjEINu3Agvee4aAgl5iN6eqhO/9agd9/U5ujoVdjqQh3RkbEgW9xKq+ppjOnn52tnSEXYqkKQV9SNoU9BKj+uMXZNV9IyOjoA+JWvQSq7pJxeQYukNWRkxBH5LDx3rIyzHGFmg2Sjm1MQW5zKgYx5u6ICsjpKUEQ3L4WA+lY/IJRiWJnNLc6hKeeGMvs+58Iqbjz5k5gUeWnp/kqiRdKOhDMhD0IrH41FV1nFY5jlhuGV+78xC/2HyQju5exhboR1wU9KE5fKxHN0tJzGZXFfPZa+bEdOzT6/bxi80HefudoyyYVpbkyiQdqI8+JG1q0UuSDEyboLtpZYCCPiRq0Uuy1E4YS1F+Dm++o6CXCAV9SCJ99Oo5k8TLzTHqJhXzloJeAgr6ELg7bZ296rqRpJldVayuGzlOQR+Co1299PW7gl6Spr66mP1Humht7w67FEkBCvoQtHX2ArorVpJn9sAFWXXfCAr6UBzu0PQHklxzqiJBr356AQV9KA5r0RFJsqqSQkqK8tikfnohxqA3s+Vmtt/MBl0hyiL+xcw2m9nrZrY4at8tZvZ28HFLogpPZ5rQTJLNzKivLuEtBb0Qe4v+IWDJKfZfB9QFH0uBbwCY2QQiK1KdS2S92LvMrHykxWYKTVEso2F29XjefOcI7rFMnCCZLKagd/dVQMspDrkB+K5HvASUmVkNcC2w0t1b3L0VWMmpf2FkBbXoZTTMqSrmSGcv+9o6wy5FQpaoPvopwK6ox03BtqG2Z7XDx3rIMRhfqBumJHnmVJcAqJ9eUudirJktNbNGM2s8cOBA2OUk1cD0B5qiWJJpdtV4APXTS8KCfjcwLerx1GDbUNvfw92XuXuDuzdUVlYmqKzUpCmKZTSUjS2gqqRQY+klYUG/AvjdYPTNecBhd98LPANcY2blwUXYa4JtWU1BL6NFUyEIxDgfvZk9DFwGVJhZE5GRNPkA7v5N4Engg8BmoAP4/WBfi5l9EXgleKp73P1UF3WzgoJeRkt9dTHfeXEHff1Obo66CrNVTEHv7jcPs9+BTw6xbzmw/P2XlrnajvUwpXxM2GVIFphdVUx3bz87mtuZVTk+7HIkJClzMTabtHWqRS+jQ4uQCGgpwVHn7uq6kVFTN6kYM7jnpxv4+nNbhj3eDP7kijqunlc1CtXJaFHQj7JjPX309GmKYhkdYwpyuf3y01m/py2m41/Z1sKP1jYp6DOMgn6U6a5YGW2fi3FRcYA/+vdG3WCVgdRHP8oU9JLK5lSXsP1gO509fWGXIgmkoB9lmoteUll9dTH9Dm+/czTsUiSBFPSj7Phc9EUKekk9A6N0Nu2LrU9f0oOCfpSp60ZS2YyJ4yjMy9FwzAyjoB9lCnpJZbk5Rl3VeM2Pk2EU9KOs7VgPZlBcpAFPkprmVJVo5E2GUdCPssPHeiguzCNH845IiqqvLubAkS5a2rvDLkUSREE/yto6eykdq24bSV26IJt5FPSjTNMfSKqr1/w4GUdBP8oU9JLqKosLKR+br6DPIAr6Uaagl1RnZsypLtYF2QyioB9lCnpJB/XVJbz1zhH6+z3sUiQBYgp6M1tiZm+a2WYz+/wg+//JzF4NPt4ys0NR+/qi9q1IZPHpaGBhcJFUNqe6mI7uPppaj4VdiiTAsIO5zSwXuB+4GmgCXjGzFe6+YeAYd/9M1PF/AiyKeopj7r4wcSWnr86ePrp7+9Wil5QXPfKmduLYkKuReMXSoj8H2OzuW929G3gEuOEUx98MPJyI4jKN5rmRdDG7SiNvMkksQT8F2BX1uCnY9h5mNh2YCTwbtbnIzBrN7CUz+/WhvoiZLQ2Oazxw4EAMZaUfTX8g6WJ8YR7TJoxhk6ZCyAiJvhh7E/Cou0dPZj3d3RuA3wL+2cxOG+xEd1/m7g3u3lBZWZngslKDgl7SyZyqErXoM0QsQb8bmBb1eGqwbTA3cVK3jbvvDv7dCjzHif33WUVz0Us6qa8uZtvBdrp6tQhJuotlZq1XgDozm0kk4G8i0jo/gZnVA+XAi1HbyoEOd+8yswrgQuDvE1F4OlKLXtJJfU0xff3OontWkmPDz800tiCXx/74AqZN0MXbVDNs0Lt7r5ndDjwD5ALL3X29md0DNLr7wJDJm4BH3D164O1c4AEz6yfy18O90aN1sk1bp4Je0scV9ZO4/fLTORbDsoId3b08/KtdvLytRUGfgmKaK9fdnwSePGnbF056fPcg570AnBlHfRnl+KgbBb2kgbEFefzptbEtLN7X7/x47R427GmDs5NcmLxvmhQ9TgeOdPHS1uaYjn296TDFhXnkaopiyTC5OUZ9TTEb9h4OuxQZhII+Tl955k2+37hr+AMDAzMDimSaeTUl/OS1Pbg7FkOfvoweBX2c9rZ1Ul9dzNd+K7bBRFUlRUmuSCQc8yaX8J8v76Sp9Zj66VOMgj5OLe1d1JQWcfoktdQlu82rKQFgw942BX2K0eyVcWo+2s3E8YVhlyESuvrqEnKMyAVZSSkK+ji4exD0BWGXIhK6MQW5zKwYx4a9CvpUo6CPw9GuXrr7+qkYpxa9CMC8yaVq0acgBX0cmo92AzBhnFr0IhDpp9996Njx6T4kNSjo49Dc3gWgrhuRwNyayKAEdd+kFgV9HA4GLfoKXYwVASJDLEFBn2oU9HFoaVfXjUi0ScVFVIwvVD99ilHQx6H5aKTrRkEv8q55k0vUok8xCvo4HDzaTXFhHkX5uWGXIpIy5tWUsHn/Ebp7+8MuRQIK+ji0tHczQRdiRU4wb3IJPX3O2/u1OlWqUNDHobm9i4nqthE5wfGpENRPnzIU9HHQ9Aci7zWzYhxF+Tnqp08hMU1qZmZLgK8SWWHqQXe/96T9vwf8A++uJfs1d38w2HcL8NfB9i+5+3cSUHdKaG7vZuG0srDLEEkpuTlGfXUJL2xu5gcxTuE9u6pYP0tJNGzQm1kucD9wNdAEvGJmKwZZEvD77n77SedOAO4CGgAHVgfntiak+hD19zst7ZrnRmQw58ycwLJVW/nzR1+P6fjSMfms/ZurydGiPEkRS4v+HGCzu28FMLNHgBuAWNZ+vRZY6e4twbkrgSXAwyMrN3UcPtZDX78zUfPciLzH55fUc8sFMzhxCenBPb1uH196YiPbmts5rXL8KFSXfWIJ+ilA9N9fTcC5gxz3ETO7BHgL+Iy77xri3CmDfREzWwosBaitrY2hrHA1BzdLqUUv8l45OcaUsjExHXvJ7Ep4YiOv7TqkoE+SRF2M/Qkww93PAlYC77sf3t2XuXuDuzdUVlYmqKzkGbhZSi16kficVjmecQW5vLbrUNilZKxYgn43MC3q8VTevegKgLs3u3tX8PBB3l0Hfthz05Va9CKJkZtjnDm1lFebtLB4ssQS9K8AdWY208wKgJuAFdEHmFlN1MPrgY3B588A15hZuZmVA9cE29Le8Ra9gl4kbgumlbFxTxtdvX1hl5KRhu2jd/deM7udSEDnAsvdfb2Z3QM0uvsK4A4zux7oBVqA3wvObTGzLxL5ZQFwz8CF2XQ30KIvH6ugF4nXwqlldPf1s2nvERZomGXCxTSO3t2fBJ48adsXoj6/E7hziHOXA8vjqDElNR/tpmxsPvm5uudMJF4D4f5a0yEFfRIopUZI0x+IJE5NaRGVxYW8qguySaGgH6Hmo90acSOSIGbGgqllGnmTJAr6EWrWXbEiCbVgailbDrTT1qn1ZhNNQT9CzUe7FPQiCTTQN/+GhlkmnIJ+BHr7+jl0rIcJ6roRSZizppYCqJ8+CRT0I9Da0YM7VKhFL5IwZWMLmFkxTv30SaCgH4Hmdk1/IJIMC6aW8lqTgj7RFPQj0HxU0x+IJMOCaWW809bFvsOdYZeSURT0I3B8nhuNoxdJqIELsuqnT6yY7oyVE707z426bkQSaV5NCfm5xm3/sRqLYQ0SA/7qQ/O49aKZSa8tnSnoR6D5aDc5BmVj8sMuRSSjFOXn8s83LuLNfbGtN7vitT385LU9CvphKOhHoLm9mwnjCrTsmUgSfOisGj50Vs3wBwL9Dt/4ny0c7eplfKHibCjqox+B5qNdGnEjkgLOP20iff3OK9syYlLcpFHQj4CmPxBJDWdPL6cgN4cXthwMu5SUpqAfgZag60ZEwlWUn8vC2jJe3NocdikpTUE/AgePdlGhETciKeH8WRNZv6eNwx2aDG0oMQW9mS0xszfNbLOZfX6Q/Z81sw1m9rqZ/beZTY/a12dmrwYfK04+N9109fZxpLNXY+hFUsT5p03EHV7eplb9UIYNejPLBe4HrgPmATeb2byTDlsLNLj7WcCjwN9H7Tvm7guDj+sTVHdoWtsjrQaNoRdJDYtqyyjMy1H3zSnEMh7pHGCzu28FMLNHgBuADQMHuPvPo45/Cfh4IoscDR3dvfT0+bDH7WzpAFAfvUiKKMzL5ezp5by4RUE/lFiCfgqwK+pxE3DuKY6/FXgq6nGRmTUSWTj8Xnf/8WAnmdlSYClAbW1tDGUlztqdrXzkGy/QP3zOHzepRC16kVRx/qyJ/OPKtzRQYggJvcPAzD4ONACXRm2e7u67zWwW8KyZveHuW04+192XAcsAGhoa3kfkxm/VWwdx4K8+ODemm6CKC/NYOFULGIukigtOn8g/roSXtzZz3Zmx3WyVTWIJ+t3AtKjHU4NtJzCzq4C/Ai51966B7e6+O/h3q5k9BywC3hP0YVq9s5U5VcX84SWzwi5FREbgrKlljC3I5UUF/aBiGXXzClBnZjPNrAC4CThh9IyZLQIeAK539/1R28vNrDD4vAK4kKi+/VTQ3++s3dnKotrysEsRkRHKz82hYcYE9dMPYdgWvbv3mtntwDNALrDc3deb2T1Ao7uvAP4BGA/80CJTzu0MRtjMBR4ws34iv1TudfeUCvrNB45ypLOXs6cr6EXS2fmzJvLlpzfxNz9eR24MXbBjCnK544o6xhTkjkJ14Yqpj97dnwSePGnbF6I+v2qI814AzoynwGRbvaMVQEEvkuaumV/Fd17YzorX9gx7rLvT1tnLjIljufEDozv4IwxZP93bmh2tTBhXwIyJY8MuRUTicFrleF76yytjOtbduewrz/HT1/dmRdBn/RQIq3e2sri2DItllQMRyQhmxofPquGFLc3HFxLKZFkd9K3t3Ww90M5idduIZJ0PnzWZvn7nqXX7wi4l6bI66NfuivTPL9aIG5GsU19dzGmV4/jp68P36ae7rA761Ttayc0xFujmJ5GsE+m+mczL21rY39YZdjlJlfVBP39ySVYMrxKR9/q1BTW4w5Nv7A27lKTK2qDv7evntV2H1W0jksVOn1RMfXUxP31dQZ+RNu07wrGePl2IFclyHz6rhsYdrew5dCzsUpIma4NeN0qJCERG30Bmd99k7Q1Ta3a2UlVSyOTSorBLEZEQzagYxxlTSrj/55tjDvuGGRO487r6tLn/JqOC/mMPvEhXb39Mx779zhEum1OZNv9RIpI8n7lqNg+9sD2mY9uO9bBs1VbOnTmBK+dWJbewBMmooC8dk92Ew7UAAAYZSURBVE93jEF/zswJ/M55M5JbkIikhSvnVsUc2j19/VzzT6v4u6c2censSvJyU78HPKOC/lu/2xB2CSKS4fJzc/iLJXO47T/W8MPVTdx8TurPlZP6v4pERFLMtfOraZhezn0r36K9qzfscoaloBcReZ/MjDs/OJcDR7p48PltYZczrIzquhERGS1nTy/nujOqeWDVFj50VjUlY/KHPccwKsYXjPogkJiC3syWAF8lssLUg+5+70n7C4HvAmcDzcCN7r492HcncCvQB9zh7s8krHoRkRD9+ZJ6Vm54h6vuWxXzOWdMKeFvf+NMzhrFObaGDXozywXuB64GmoBXzGzFSUsC3gq0uvvpZnYT8GXgRjObR2SN2fnAZOD/mdlsd+9L9DciIjLaZlaM44e3nc/6PW0xHd/R3cu3nt/GDff/kt89bzqfu3YOJUXD/yUQr1ha9OcAm919K4CZPQLcwImLfN8A3B18/ijwNYv8bXID8Ii7dwHbzGxz8HwvJqZ8EZFwLaotZ9H7mDPrpnNque9nb/HdF7fz41f3MKm48Pi+8rEF/OC28xNeYyxBPwXYFfW4CTh3qGOCxcQPAxOD7S+ddO6Uwb6ImS0FlgLU1qb+cCURkZEoKcrn7uvn85HFU/m3X26js7fvhH3JkDIXY919GbAMoKGhwUMuR0Qkqc6cWsp9Ny4cla8Vy/DK3cC0qMdTg22DHmNmeUApkYuysZwrIiJJFEvQvwLUmdlMMysgcnF1xUnHrABuCT7/KPCsu3uw/SYzKzSzmUAd8KvElC4iIrEYtusm6HO/HXiGyPDK5e6+3szuARrdfQXwbeDfg4utLUR+GRAc9wMiF257gU9qxI2IyOiySMM7tTQ0NHhjY2PYZYiIpA0zW+3ug074pSkQREQynIJeRCTDKehFRDKcgl5EJMOl5MVYMzsA7Bjh6RXAwQSWk4n0Gg1Pr1Fs9DoNb7Reo+nuXjnYjpQM+niYWeNQV54lQq/R8PQaxUav0/BS4TVS142ISIZT0IuIZLhMDPplYReQBvQaDU+vUWz0Og0v9Nco4/roRUTkRJnYohcRkSgKehGRDJcxQW9mS8zsTTPbbGafD7ueVGFm08zs52a2wczWm9mngu0TzGylmb0d/Bv7WmgZysxyzWytmf00eDzTzF4O3lPfD6bpzlpmVmZmj5rZJjPbaGbn6330Xmb2meBnbZ2ZPWxmRWG/lzIi6KMWML8OmAfcHCxMLpHpoT/n7vOA84BPBq/N54H/dvc64L+Dx9nuU8DGqMdfBv7J3U8HWoFbQ6kqdXwVeNrd64EFRF4rvY+imNkU4A6gwd3PIDK1+02E/F7KiKAnagFzd+8GBhYwz3ruvtfd1wSfHyHywzmFyOvzneCw7wC/Hk6FqcHMpgIfAh4MHhtwBZHF7iHLXyMzKwUuIbL2BO7e7e6H0PtoMHnAmGC1vbHAXkJ+L2VK0A+2gPmgi5BnMzObASwCXgaq3H1vsGsfUBVSWanin4E/B/qDxxOBQ+7eGzzO9vfUTOAA8G9B99aDZjYOvY9O4O67ga8AO4kE/GFgNSG/lzIl6GUYZjYeeAz4tLu3Re8Lln3M2nG2ZvZhYL+7rw67lhSWBywGvuHui4B2Tuqmyfb3EUBwjeIGIr8YJwPjgCWhFkXmBL0WIT8FM8snEvL/6e6PB5vfMbOaYH8NsD+s+lLAhcD1ZradSLffFUT6o8uCP79B76kmoMndXw4eP0ok+PU+OtFVwDZ3P+DuPcDjRN5fob6XMiXoY1nAPCsFfc3fBja6+31Ru6IXdL8F+K/Rri1VuPud7j7V3WcQee886+6/DfycyGL3oNdoH7DLzOYEm64ksha03kcn2gmcZ2Zjg5+9gdcp1PdSxtwZa2YfJNLPOrCA+f8NuaSUYGYXAc8Db/Bu//NfEumn/wFQS2RK6I+5e0soRaYQM7sM+FN3/7CZzSLSwp8ArAU+7u5dYdYXJjNbSORidQGwFfh9Io1FvY+imNn/AW4kMuJtLfAJIn3yob2XMiboRURkcJnSdSMiIkNQ0IuIZDgFvYhIhlPQi4hkOAW9iEiGU9CLiGQ4Bb2ISIb7/6tepIxGEyENAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AtRPI8ocIRai",
        "outputId": "9e0bb50c-cda0-4cd9-8c08-162bb5859392"
      },
      "source": [
        "trainer.logged_metrics"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'avg_train_accuracy': 0.7865777611732483,\n",
              " 'avg_train_loss': 0.042027246206998825,\n",
              " 'avg_val_f1': 0.5355244874954224,\n",
              " 'avg_val_loss': 0.09359695762395859,\n",
              " 'loss_epoch': 0.16770172119140625,\n",
              " 'loss_step': tensor(0.2190),\n",
              " 'train_f1_epoch': 0.7867752909660339,\n",
              " 'train_f1_step': tensor(0.7619),\n",
              " 'val_f1_epoch': 0.5398086309432983,\n",
              " 'val_f1_step': 0.0,\n",
              " 'val_loss_epoch': 0.09398338943719864,\n",
              " 'val_loss_step': 0.04529694840312004}"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTYeeBp4LDuy"
      },
      "source": [
        "%reload_ext tensorboard\n",
        "%tensorboard --logdir lightning_logs/\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O5q2zeY3zLzG"
      },
      "source": [
        "# Saving only the pytorch model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GeNvSmBCwWpN"
      },
      "source": [
        "# saving model weights\n",
        "saved_model_pth = \"saved_model.pth\"\n",
        "\n",
        "torch.save(toxic_comment_model.pytorch_model.state_dict(), saved_model_pth)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z_Qa6mk9bB-V"
      },
      "source": [
        "# Exporting model as pure pytorch "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aSX1sSa1hsMI"
      },
      "source": [
        "new_bert_model = BertModel(6)\n",
        "\n",
        "new_bert_model.load_state_dict(torch.load(saved_model_pth))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytqP_Olebu64"
      },
      "source": [
        "# check values are the same\n",
        "\n",
        "inf_preds = new_bert_model(sample_batch[\"input_ids\"], sample_batch[\"attention_mask\"])\n",
        "print(inf_preds.shape) # should be 8 x 6\n",
        "inf_preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q0vlXp_dc94H"
      },
      "source": [
        "toxic_comment_model(sample_batch[\"input_ids\"], sample_batch[\"attention_mask\"])[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_GEsu9KydC93"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}