GPT_J_6B_Topic_Modelling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GPT-J-6B Topic Modelling.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fbf4b2460d7d4dad804e7e727b6b4c59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ab2afcba1de54204a450f8d780f7cd9d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f97e3f83eec64b9da6426a1663d66b69",
              "IPY_MODEL_bb49011c43f64924adaa20aaec136aee"
            ]
          }
        },
        "ab2afcba1de54204a450f8d780f7cd9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f97e3f83eec64b9da6426a1663d66b69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb828710850741b58ed357ef2551179c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1042301,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1042301,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a7a31a0d1f34e329b0004388fa3e653"
          }
        },
        "bb49011c43f64924adaa20aaec136aee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3c24c5146ed84ad6a13cecac908d77b2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.04M/1.04M [00:01&lt;00:00, 624kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_59154f40ef8846dda76016a0c6496343"
          }
        },
        "cb828710850741b58ed357ef2551179c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a7a31a0d1f34e329b0004388fa3e653": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c24c5146ed84ad6a13cecac908d77b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "59154f40ef8846dda76016a0c6496343": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc508e48c9754c9c86328d336f8b8a38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b4cd19fb39354e0586737b2323ef7470",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0678f4d1926a4a73915bc5a86e031e37",
              "IPY_MODEL_b08f63d6ea6642b5af67f1f7ffc23bbe"
            ]
          }
        },
        "b4cd19fb39354e0586737b2323ef7470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0678f4d1926a4a73915bc5a86e031e37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5fc5e9c458f4469e9f2b5b3b1f92763c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aea7b69157614da791d66b3d14bbfab6"
          }
        },
        "b08f63d6ea6642b5af67f1f7ffc23bbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d49ace7f94e5493c8c266951df508bcd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 972kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d0ec6249898e46b4ae5085a1d60cb6d2"
          }
        },
        "5fc5e9c458f4469e9f2b5b3b1f92763c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aea7b69157614da791d66b3d14bbfab6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d49ace7f94e5493c8c266951df508bcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d0ec6249898e46b4ae5085a1d60cb6d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f085f95f4f2b41d39ccadd41d51563c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_88d146ac80b441b8b86e534419d852c9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f1496cdd9a3e432d8a03d82ad47f275d",
              "IPY_MODEL_bbd1da114e054943a0e90ca00a522e14"
            ]
          }
        },
        "88d146ac80b441b8b86e534419d852c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1496cdd9a3e432d8a03d82ad47f275d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9e815b652ed443e7b0a21f66282b4db9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1355256,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1355256,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_53df321d7fbc4a959913d15548bbb577"
          }
        },
        "bbd1da114e054943a0e90ca00a522e14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f65669d413a84b52a99ea47ffd10df86",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 1.45MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7649a02849bd4bcebf30a57ac2afe4d3"
          }
        },
        "9e815b652ed443e7b0a21f66282b4db9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "53df321d7fbc4a959913d15548bbb577": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f65669d413a84b52a99ea47ffd10df86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7649a02849bd4bcebf30a57ac2afe4d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab283591978f478f90382624b50b5c99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02a9b039d15640a4babc86068f5f3382",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_404a8a74ddb74b668330e3ac109ad936",
              "IPY_MODEL_f79535e1fe9042b0811e75c32f1103b3"
            ]
          }
        },
        "02a9b039d15640a4babc86068f5f3382": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "404a8a74ddb74b668330e3ac109ad936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_45ec2f2169f64bbaabc1da59af92010f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 736,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 736,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c19f935f197548b182e5e2ec1a845b20"
          }
        },
        "f79535e1fe9042b0811e75c32f1103b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7724d7e460f74d99bb8850dd19635155",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 736/736 [03:12&lt;00:00, 3.82B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b955aa83ae574583b2164fd3dcbe8dc0"
          }
        },
        "45ec2f2169f64bbaabc1da59af92010f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c19f935f197548b182e5e2ec1a845b20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7724d7e460f74d99bb8850dd19635155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b955aa83ae574583b2164fd3dcbe8dc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d020c2a260604b3abce3ba0330ad82bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7ebf8dbff8204bac906791141d811e2f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_59840c160c2d4fe083287a1c7cf14f01",
              "IPY_MODEL_d50d279e12d743c6b4230277c25962fa"
            ]
          }
        },
        "7ebf8dbff8204bac906791141d811e2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59840c160c2d4fe083287a1c7cf14f01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_99f7eb3fd674409a88c11a62accfed6f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3721,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3721,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_54241adcb36a4c37b2b05a17aecd7380"
          }
        },
        "d50d279e12d743c6b4230277c25962fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_09a48d61d2c84fb689ce17be4debac38",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.72k/3.72k [00:00&lt;00:00, 4.36kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e3f9c8ef9254155829342977206b42f"
          }
        },
        "99f7eb3fd674409a88c11a62accfed6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "54241adcb36a4c37b2b05a17aecd7380": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09a48d61d2c84fb689ce17be4debac38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e3f9c8ef9254155829342977206b42f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f995731eb92f4121a920ed4c6622da86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0a1c296f0fb843eda0a9ee41e07cafac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7a559bac6132437782b3e92cfe80b1d2",
              "IPY_MODEL_cac3a28737b94d64a5b37984bfab04b5"
            ]
          }
        },
        "0a1c296f0fb843eda0a9ee41e07cafac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a559bac6132437782b3e92cfe80b1d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c0e32f91a58641a1b54e7c93333e2fa9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 686,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 686,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fe7ce0e260104613a68fedda4babe779"
          }
        },
        "cac3a28737b94d64a5b37984bfab04b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_914dd6d5ef89412e8f5938af87580f13",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 686/686 [00:00&lt;00:00, 1.67kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fa89483651f1418d96bcaa530f9679bf"
          }
        },
        "c0e32f91a58641a1b54e7c93333e2fa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fe7ce0e260104613a68fedda4babe779": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "914dd6d5ef89412e8f5938af87580f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fa89483651f1418d96bcaa530f9679bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8319f91b448748cfadcb5e308df88ea4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_11507eaa281c45069bd7d9aa212fa8a8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bea259f564e8420994d3343a78bd5d92",
              "IPY_MODEL_479ce3a697554b80825dc81d0911bb38"
            ]
          }
        },
        "11507eaa281c45069bd7d9aa212fa8a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bea259f564e8420994d3343a78bd5d92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f2196b351e4e4ee4939905c07c13fffe",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 122,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 122,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0711d4ad8190486389ed4b2395757e81"
          }
        },
        "479ce3a697554b80825dc81d0911bb38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_395811ef31e3437d83485c8ad39265a6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 122/122 [00:01&lt;00:00, 83.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de22013fe820480e9a1d4dc09e4cc761"
          }
        },
        "f2196b351e4e4ee4939905c07c13fffe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0711d4ad8190486389ed4b2395757e81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "395811ef31e3437d83485c8ad39265a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de22013fe820480e9a1d4dc09e4cc761": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da696617d67443f38865fa1d02990543": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_29801af3d4fe4ccf9de84b98e35cb345",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_49692a8a42404f658afa591fbdac10a8",
              "IPY_MODEL_88b8e9ae983d4a2fa04fe183a5e08501"
            ]
          }
        },
        "29801af3d4fe4ccf9de84b98e35cb345": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "49692a8a42404f658afa591fbdac10a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0cb46b25cfff462995e3e81d1c4d907a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456356,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456356,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2b8fb883c8be4cd28f17ff8e0cc14966"
          }
        },
        "88b8e9ae983d4a2fa04fe183a5e08501": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7477d052a4b242a9ab452c5ab270fe2e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:01&lt;00:00, 446kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_537a37aa0e9d40c7a1716226d34dd278"
          }
        },
        "0cb46b25cfff462995e3e81d1c4d907a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2b8fb883c8be4cd28f17ff8e0cc14966": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7477d052a4b242a9ab452c5ab270fe2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "537a37aa0e9d40c7a1716226d34dd278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "146a4f5514ab4d6f960873729fef517f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_849338533ab24feda2e5b672bbd0728d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e7d7f054cfdd4d699b7c8b5ab6f7c432",
              "IPY_MODEL_98d56955455a4618a117c8251504b141"
            ]
          }
        },
        "849338533ab24feda2e5b672bbd0728d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e7d7f054cfdd4d699b7c8b5ab6f7c432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_41df3185c5c14a148394847bde90575c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 229,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 229,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b9218bbda6104eb49aabc2c3993848ea"
          }
        },
        "98d56955455a4618a117c8251504b141": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ef7f7be28b0948bbbba3ed52ca4fbb22",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 229/229 [00:00&lt;00:00, 452B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f166522c6f244548e0fc4412f1cf56d"
          }
        },
        "41df3185c5c14a148394847bde90575c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b9218bbda6104eb49aabc2c3993848ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef7f7be28b0948bbbba3ed52ca4fbb22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f166522c6f244548e0fc4412f1cf56d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f72ad09c93df4e6fb6147a295bbecf58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6b2f9adb62e44936baf4e5b7219bd77c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ef86a8bf9da142689cbeed5ec2261b86",
              "IPY_MODEL_61efd84f33334eb6815a41eb53f22914"
            ]
          }
        },
        "6b2f9adb62e44936baf4e5b7219bd77c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef86a8bf9da142689cbeed5ec2261b86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_46e8b5c065f543cfb3add6839ce3d60f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 328515953,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 328515953,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9b8e9c8945f45eeaf33fe9e26d1dc76"
          }
        },
        "61efd84f33334eb6815a41eb53f22914": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_25b6eacdb5b3425dbd09e47da187b0c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 329M/329M [00:09&lt;00:00, 33.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_85ff2ba5203b4a62a3154ab7cacf0c5f"
          }
        },
        "46e8b5c065f543cfb3add6839ce3d60f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9b8e9c8945f45eeaf33fe9e26d1dc76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25b6eacdb5b3425dbd09e47da187b0c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "85ff2ba5203b4a62a3154ab7cacf0c5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a99debc45d54080ab32a03d8e27ac96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_715747079ead41daae87c9a6e45fbf5f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_19fc43e759504de6878953fef71fc4d1",
              "IPY_MODEL_af612ad2dbcc48ce82a2fec347165787"
            ]
          }
        },
        "715747079ead41daae87c9a6e45fbf5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "19fc43e759504de6878953fef71fc4d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b53c7e66eab6421b864e1f4efdeffda3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 53,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 53,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8ee100d28cab4898902823690809f3f9"
          }
        },
        "af612ad2dbcc48ce82a2fec347165787": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80e0af1fbe9c47f39bd6da7c54f0a133",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 53.0/53.0 [00:00&lt;00:00, 63.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d0dae66202d44fa8df3f9b17f0ea21f"
          }
        },
        "b53c7e66eab6421b864e1f4efdeffda3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8ee100d28cab4898902823690809f3f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80e0af1fbe9c47f39bd6da7c54f0a133": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d0dae66202d44fa8df3f9b17f0ea21f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4c4fa8321fc0483bbb50a99da0064237": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_07248052e1d543729b34fb3a4a4c4418",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1df916cccb7a4b55bebb3cbe86ee7f92",
              "IPY_MODEL_ed62fe818f4b46c3a1ca02050b61780a"
            ]
          }
        },
        "07248052e1d543729b34fb3a4a4c4418": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1df916cccb7a4b55bebb3cbe86ee7f92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_25a8df472e0c4c9cae5655fad29a18c3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 239,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 239,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9626f232541d4102bc25423df412f872"
          }
        },
        "ed62fe818f4b46c3a1ca02050b61780a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ca6f4d24e07e44b8ad9907014389f680",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 239/239 [00:00&lt;00:00, 614B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_281c561552c547b0b98fee51ce346da0"
          }
        },
        "25a8df472e0c4c9cae5655fad29a18c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9626f232541d4102bc25423df412f872": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca6f4d24e07e44b8ad9907014389f680": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "281c561552c547b0b98fee51ce346da0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a1274fdf8a494752ab8b22d377d5223b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7bd8131fa5fc40f7b3b60de6b8827039",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_52b98a05ca9c43208721203c3b42f3cd",
              "IPY_MODEL_ccb082b637104a97bd3a92d97dc2ad30"
            ]
          }
        },
        "7bd8131fa5fc40f7b3b60de6b8827039": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52b98a05ca9c43208721203c3b42f3cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ee473403a6e402db6f1d507e51b3f7d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1355881,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1355881,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18e91362094e4caeb248c18ade77faf4"
          }
        },
        "ccb082b637104a97bd3a92d97dc2ad30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d266a10b4c16411b92a177d8ae06c905",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.36M/1.36M [00:01&lt;00:00, 863kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2103d2a869640c0a5450b6d1a2f3612"
          }
        },
        "5ee473403a6e402db6f1d507e51b3f7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18e91362094e4caeb248c18ade77faf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d266a10b4c16411b92a177d8ae06c905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2103d2a869640c0a5450b6d1a2f3612": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "866b87e3ef8e40519f9474bda1475474": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a6bc1ee1fd1d4a0eb8c29aadf2dbf444",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e000b07793e74d7f8951dd98215b499c",
              "IPY_MODEL_0b142d3e34894931ad4a12a755810e3e"
            ]
          }
        },
        "a6bc1ee1fd1d4a0eb8c29aadf2dbf444": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e000b07793e74d7f8951dd98215b499c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ef5ee2c450424860bb2e8067f46753fd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1123,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1123,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ba798647eea46dcb8a51e30764a934e"
          }
        },
        "0b142d3e34894931ad4a12a755810e3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af97567fc4204c129dfeb2fb593ed4fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.12k/1.12k [00:01&lt;00:00, 1.12kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7a2e95a96a594e72a69843eea0dee287"
          }
        },
        "ef5ee2c450424860bb2e8067f46753fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ba798647eea46dcb8a51e30764a934e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af97567fc4204c129dfeb2fb593ed4fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7a2e95a96a594e72a69843eea0dee287": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61e2a230db48484e80de7f1728b5d4bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_50138ae3a8b74213a6a25823c4f2c154",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c73a942bfca845d080974dcf5f5ae966",
              "IPY_MODEL_693d8a5bf2ff40f7a017268fa2c0bbf4"
            ]
          }
        },
        "50138ae3a8b74213a6a25823c4f2c154": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c73a942bfca845d080974dcf5f5ae966": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eca62b86260c4c379e11fac6dde84ac2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 798293,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 798293,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2b4a4f3739c24a8e8062b6a2f825942b"
          }
        },
        "693d8a5bf2ff40f7a017268fa2c0bbf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f5b20e506f5a4b438205e00259718199",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 798k/798k [00:00&lt;00:00, 1.44MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_959df93b8b4a4c2fb6168543fbfe888a"
          }
        },
        "eca62b86260c4c379e11fac6dde84ac2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2b4a4f3739c24a8e8062b6a2f825942b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5b20e506f5a4b438205e00259718199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "959df93b8b4a4c2fb6168543fbfe888a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bc45d04ae8884f26a55774201f0942ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_97eda0a864c24f589c316f5ad9e086c8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8f15f816e5aa492d8e6a6f5bfe46334f",
              "IPY_MODEL_479454e31211411b98e83a9e4494d325"
            ]
          }
        },
        "97eda0a864c24f589c316f5ad9e086c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f15f816e5aa492d8e6a6f5bfe46334f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_218f9b546c064621883e33f9dd9241ae",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 190,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 190,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e7bd0d57ed1c4c73866efea49333e5b1"
          }
        },
        "479454e31211411b98e83a9e4494d325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ad758c4162d64280b4e26d49b14a8cdd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 190/190 [00:01&lt;00:00, 112B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5e9d3995cd53456d82d0df551207f009"
          }
        },
        "218f9b546c064621883e33f9dd9241ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e7bd0d57ed1c4c73866efea49333e5b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad758c4162d64280b4e26d49b14a8cdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5e9d3995cd53456d82d0df551207f009": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/maxmatical/ml-cheatsheet/blob/master/GPT_J_6B_Topic_Modelling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pHIJVqHsh4An"
      },
      "source": [
        "# GPT-J-6B Inference Demo\n",
        "\n",
        "<a href=\"http://colab.research.google.com/github/kingoflolz/mesh-transformer-jax/blob/master/colab_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
        "\n",
        "This notebook demonstrates how to run the [GPT-J-6B model](https://github.com/kingoflolz/mesh-transformer-jax/#GPT-J-6B). See the link for more details about the model, including evaluation metrics and credits."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CMw_dSQKfhT"
      },
      "source": [
        "## Install Dependencies\n",
        "\n",
        "First we download the model and install some dependencies. This step takes at least 5 minutes (possibly longer depending on server load).\n",
        "\n",
        "!!! **Make sure you are using a TPU runtime!** !!!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n7xAFw-LOYfe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e11b6331-f140-4ef6-c698-863d397848ef"
      },
      "source": [
        "!apt install zstd\n",
        "\n",
        "# the \"slim\" version contain only bf16 weights and no optimizer parameters, which minimizes bandwidth and memory\n",
        "!time wget https://the-eye.eu/public/AI/GPT-J-6B/step_383500_slim.tar.zstd\n",
        "# for full fp32 model w/ optimizer params, 61gb\n",
        "# https://the-eye.eu/public/AI/GPT-J-6B/step_383500.tar.zstd \n",
        "\n",
        "!time tar -I zstd -xf step_383500_slim.tar.zstd\n",
        "\n",
        "!git clone https://github.com/kingoflolz/mesh-transformer-jax.git\n",
        "!pip install -r mesh-transformer-jax/requirements.txt\n",
        "\n",
        "# jax 0.2.12 is required due to a regression with xmap in 0.2.13\n",
        "!pip install mesh-transformer-jax/ jax==0.2.12"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  zstd\n",
            "0 upgraded, 1 newly installed, 0 to remove and 40 not upgraded.\n",
            "Need to get 278 kB of archives.\n",
            "After this operation, 1,141 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 zstd amd64 1.3.3+dfsg-2ubuntu1.2 [278 kB]\n",
            "Fetched 278 kB in 1s (299 kB/s)\n",
            "Selecting previously unselected package zstd.\n",
            "(Reading database ... 160837 files and directories currently installed.)\n",
            "Preparing to unpack .../zstd_1.3.3+dfsg-2ubuntu1.2_amd64.deb ...\n",
            "Unpacking zstd (1.3.3+dfsg-2ubuntu1.2) ...\n",
            "Setting up zstd (1.3.3+dfsg-2ubuntu1.2) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "--2021-07-22 18:55:29--  https://the-eye.eu/public/AI/GPT-J-6B/step_383500_slim.tar.zstd\n",
            "Resolving the-eye.eu (the-eye.eu)... 162.213.130.242\n",
            "Connecting to the-eye.eu (the-eye.eu)|162.213.130.242|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9414712325 (8.8G) [application/octet-stream]\n",
            "Saving to: ‘step_383500_slim.tar.zstd’\n",
            "\n",
            "step_383500_slim.ta 100%[===================>]   8.77G  32.8MB/s    in 3m 19s  \n",
            "\n",
            "2021-07-22 18:58:48 (45.2 MB/s) - ‘step_383500_slim.tar.zstd’ saved [9414712325/9414712325]\n",
            "\n",
            "\n",
            "real\t3m22.122s\n",
            "user\t0m5.514s\n",
            "sys\t0m23.787s\n",
            "\n",
            "real\t3m59.515s\n",
            "user\t0m34.737s\n",
            "sys\t0m27.407s\n",
            "Cloning into 'mesh-transformer-jax'...\n",
            "remote: Enumerating objects: 630, done.\u001b[K\n",
            "remote: Counting objects: 100% (281/281), done.\u001b[K\n",
            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
            "remote: Total 630 (delta 227), reused 199 (delta 175), pack-reused 349\u001b[K\n",
            "Receiving objects: 100% (630/630), 182.74 KiB | 3.58 MiB/s, done.\n",
            "Resolving deltas: 100% (407/407), done.\n",
            "Collecting git+https://github.com/deepmind/dm-haiku (from -r mesh-transformer-jax/requirements.txt (line 10))\n",
            "  Cloning https://github.com/deepmind/dm-haiku to /tmp/pip-req-build-utjl3xeg\n",
            "  Running command git clone -q https://github.com/deepmind/dm-haiku /tmp/pip-req-build-utjl3xeg\n",
            "Collecting git+https://github.com/EleutherAI/lm-evaluation-harness/ (from -r mesh-transformer-jax/requirements.txt (line 11))\n",
            "  Cloning https://github.com/EleutherAI/lm-evaluation-harness/ to /tmp/pip-req-build-8m9fijj5\n",
            "  Running command git clone -q https://github.com/EleutherAI/lm-evaluation-harness/ /tmp/pip-req-build-8m9fijj5\n",
            "Requirement already satisfied: numpy~=1.19.5 in /usr/local/lib/python3.7/dist-packages (from -r mesh-transformer-jax/requirements.txt (line 1)) (1.19.5)\n",
            "Collecting transformers~=4.8.2\n",
            "  Downloading transformers-4.8.2-py3-none-any.whl (2.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.5 MB 5.5 MB/s \n",
            "\u001b[?25hCollecting tqdm~=4.45.0\n",
            "  Downloading tqdm-4.45.0-py2.py3-none-any.whl (60 kB)\n",
            "\u001b[K     |████████████████████████████████| 60 kB 8.5 MB/s \n",
            "\u001b[?25hCollecting setuptools~=51.3.3\n",
            "  Downloading setuptools-51.3.3-py3-none-any.whl (786 kB)\n",
            "\u001b[K     |████████████████████████████████| 786 kB 29.6 MB/s \n",
            "\u001b[?25hCollecting wandb~=0.10.22\n",
            "  Downloading wandb-0.10.33-py2.py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 43.3 MB/s \n",
            "\u001b[?25hCollecting einops~=0.3.0\n",
            "  Downloading einops-0.3.0-py2.py3-none-any.whl (25 kB)\n",
            "Collecting requests~=2.25.1\n",
            "  Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 9.0 MB/s \n",
            "\u001b[?25hCollecting fabric~=2.6.0\n",
            "  Downloading fabric-2.6.0-py2.py3-none-any.whl (53 kB)\n",
            "\u001b[K     |████████████████████████████████| 53 kB 1.4 MB/s \n",
            "\u001b[?25hCollecting optax==0.0.6\n",
            "  Downloading optax-0.0.6-py3-none-any.whl (96 kB)\n",
            "\u001b[K     |████████████████████████████████| 96 kB 5.7 MB/s \n",
            "\u001b[?25hCollecting ray==1.4.1\n",
            "  Downloading ray-1.4.1-cp37-cp37m-manylinux2014_x86_64.whl (51.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 51.6 MB 49 kB/s \n",
            "\u001b[?25hRequirement already satisfied: jax~=0.2.12 in /usr/local/lib/python3.7/dist-packages (from -r mesh-transformer-jax/requirements.txt (line 13)) (0.2.17)\n",
            "Requirement already satisfied: Flask~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from -r mesh-transformer-jax/requirements.txt (line 14)) (1.1.4)\n",
            "Requirement already satisfied: cloudpickle~=1.3.0 in /usr/local/lib/python3.7/dist-packages (from -r mesh-transformer-jax/requirements.txt (line 15)) (1.3.0)\n",
            "Collecting tensorflow-cpu~=2.5.0\n",
            "  Downloading tensorflow_cpu-2.5.0-cp37-cp37m-manylinux2010_x86_64.whl (168.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 168.3 MB 15 kB/s \n",
            "\u001b[?25hCollecting google-cloud-storage~=1.36.2\n",
            "  Downloading google_cloud_storage-1.36.2-py2.py3-none-any.whl (97 kB)\n",
            "\u001b[K     |████████████████████████████████| 97 kB 7.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: smart_open[gcs] in /usr/local/lib/python3.7/dist-packages (from -r mesh-transformer-jax/requirements.txt (line 18)) (5.1.0)\n",
            "Collecting func_timeout\n",
            "  Downloading func_timeout-4.3.5.tar.gz (44 kB)\n",
            "\u001b[K     |████████████████████████████████| 44 kB 2.6 MB/s \n",
            "\u001b[?25hCollecting ftfy\n",
            "  Downloading ftfy-6.0.3.tar.gz (64 kB)\n",
            "\u001b[K     |████████████████████████████████| 64 kB 2.9 MB/s \n",
            "\u001b[?25hCollecting fastapi\n",
            "  Downloading fastapi-0.67.0-py3-none-any.whl (51 kB)\n",
            "\u001b[K     |████████████████████████████████| 51 kB 375 kB/s \n",
            "\u001b[?25hCollecting uvicorn\n",
            "  Downloading uvicorn-0.14.0-py3-none-any.whl (50 kB)\n",
            "\u001b[K     |████████████████████████████████| 50 kB 6.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from dm-haiku==0.0.5.dev0->-r mesh-transformer-jax/requirements.txt (line 10)) (0.12.0)\n",
            "Collecting jmp>=0.0.2\n",
            "  Downloading jmp-0.0.2-py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from dm-haiku==0.0.5.dev0->-r mesh-transformer-jax/requirements.txt (line 10)) (0.8.9)\n",
            "Requirement already satisfied: typing_extensions in /usr/local/lib/python3.7/dist-packages (from dm-haiku==0.0.5.dev0->-r mesh-transformer-jax/requirements.txt (line 10)) (3.7.4.3)\n",
            "Collecting black==20.8b1\n",
            "  Downloading black-20.8b1.tar.gz (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 66.3 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting best_download>=0.0.6\n",
            "  Downloading best_download-0.0.7-py3-none-any.whl (4.5 kB)\n",
            "Collecting datasets>=1.2.1\n",
            "  Downloading datasets-1.10.2-py3-none-any.whl (542 kB)\n",
            "\u001b[K     |████████████████████████████████| 542 kB 61.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click>=7.1 in /usr/local/lib/python3.7/dist-packages (from lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (7.1.2)\n",
            "Collecting scikit-learn>=0.24.1\n",
            "  Downloading scikit_learn-0.24.2-cp37-cp37m-manylinux2010_x86_64.whl (22.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 22.3 MB 54 kB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.7 in /usr/local/lib/python3.7/dist-packages (from lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.9.0+cu102)\n",
            "Collecting sqlitedict==1.6.0\n",
            "  Downloading sqlitedict-1.6.0.tar.gz (29 kB)\n",
            "Collecting pytablewriter==0.58.0\n",
            "  Downloading pytablewriter-0.58.0-py3-none-any.whl (96 kB)\n",
            "\u001b[K     |████████████████████████████████| 96 kB 7.2 MB/s \n",
            "\u001b[?25hCollecting sacrebleu==1.5.0\n",
            "  Downloading sacrebleu-1.5.0-py3-none-any.whl (65 kB)\n",
            "\u001b[K     |████████████████████████████████| 65 kB 4.9 MB/s \n",
            "\u001b[?25hCollecting pycountry==20.7.3\n",
            "  Downloading pycountry-20.7.3.tar.gz (10.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 10.1 MB 81.2 MB/s \n",
            "\u001b[?25hCollecting numexpr==2.7.2\n",
            "  Downloading numexpr-2.7.2-cp37-cp37m-manylinux2010_x86_64.whl (471 kB)\n",
            "\u001b[K     |████████████████████████████████| 471 kB 55.4 MB/s \n",
            "\u001b[?25hCollecting lm_dataformat==0.0.19\n",
            "  Downloading lm_dataformat-0.0.19-py3-none-any.whl (5.4 kB)\n",
            "Collecting pytest==6.2.3\n",
            "  Downloading pytest-6.2.3-py3-none-any.whl (280 kB)\n",
            "\u001b[K     |████████████████████████████████| 280 kB 80.1 MB/s \n",
            "\u001b[?25hCollecting pybind11==2.6.2\n",
            "  Downloading pybind11-2.6.2-py2.py3-none-any.whl (191 kB)\n",
            "\u001b[K     |████████████████████████████████| 191 kB 77.4 MB/s \n",
            "\u001b[?25hCollecting tqdm-multiprocess==0.0.11\n",
            "  Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n",
            "Collecting zstandard==0.15.2\n",
            "  Downloading zstandard-0.15.2-cp37-cp37m-manylinux2014_x86_64.whl (2.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.2 MB 69.7 MB/s \n",
            "\u001b[?25hCollecting jsonlines==2.0.0\n",
            "  Downloading jsonlines-2.0.0-py3-none-any.whl (6.3 kB)\n",
            "Collecting mock==4.0.3\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Collecting openai==0.6.4\n",
            "  Downloading openai-0.6.4.tar.gz (159 kB)\n",
            "\u001b[K     |████████████████████████████████| 159 kB 81.5 MB/s \n",
            "\u001b[?25hCollecting chex>=0.0.4\n",
            "  Downloading chex-0.0.8-py3-none-any.whl (57 kB)\n",
            "\u001b[K     |████████████████████████████████| 57 kB 5.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: jaxlib>=0.1.37 in /usr/local/lib/python3.7/dist-packages (from optax==0.0.6->-r mesh-transformer-jax/requirements.txt (line 9)) (0.1.69+cuda110)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 68.6 MB/s \n",
            "\u001b[?25hCollecting opencensus\n",
            "  Downloading opencensus-0.7.13-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 85.7 MB/s \n",
            "\u001b[?25hCollecting redis>=3.5.0\n",
            "  Downloading redis-3.5.3-py2.py3-none-any.whl (72 kB)\n",
            "\u001b[K     |████████████████████████████████| 72 kB 590 kB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.15.3 in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (3.17.3)\n",
            "Collecting aiohttp-cors\n",
            "  Downloading aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting pydantic>=1.8\n",
            "  Downloading pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl (10.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 10.1 MB 74.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (2.6.0)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (1.34.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (3.13)\n",
            "Requirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (0.11.0)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (1.0.2)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Collecting py-spy>=0.2.0\n",
            "  Downloading py_spy-0.3.7-py2.py3-none-manylinux1_x86_64.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 70.4 MB/s \n",
            "\u001b[?25hCollecting gpustat\n",
            "  Downloading gpustat-0.6.0.tar.gz (78 kB)\n",
            "\u001b[K     |████████████████████████████████| 78 kB 6.8 MB/s \n",
            "\u001b[?25hCollecting aioredis\n",
            "  Downloading aioredis-1.3.1-py3-none-any.whl (65 kB)\n",
            "\u001b[K     |████████████████████████████████| 65 kB 4.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (3.0.12)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.7/dist-packages (from black==20.8b1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.4.4)\n",
            "Collecting pathspec<1,>=0.6\n",
            "  Downloading pathspec-0.9.0-py2.py3-none-any.whl (31 kB)\n",
            "Collecting typed-ast>=1.4.0\n",
            "  Downloading typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl (743 kB)\n",
            "\u001b[K     |████████████████████████████████| 743 kB 70.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: toml>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from black==20.8b1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (0.10.2)\n",
            "Collecting mypy-extensions>=0.4.3\n",
            "  Downloading mypy_extensions-0.4.3-py2.py3-none-any.whl (4.5 kB)\n",
            "Collecting regex>=2020.1.8\n",
            "  Downloading regex-2021.7.6-cp37-cp37m-manylinux2014_x86_64.whl (721 kB)\n",
            "\u001b[K     |████████████████████████████████| 721 kB 65.7 MB/s \n",
            "\u001b[?25hCollecting ujson\n",
            "  Downloading ujson-4.0.2-cp37-cp37m-manylinux1_x86_64.whl (179 kB)\n",
            "\u001b[K     |████████████████████████████████| 179 kB 76.8 MB/s \n",
            "\u001b[?25hCollecting tabledata<2,>=1.1.3\n",
            "  Downloading tabledata-1.2.0-py3-none-any.whl (11 kB)\n",
            "Collecting DataProperty<2,>=0.50.0\n",
            "  Downloading DataProperty-0.52.0-py3-none-any.whl (26 kB)\n",
            "Collecting msgfy<1,>=0.1.0\n",
            "  Downloading msgfy-0.1.0-py3-none-any.whl (4.3 kB)\n",
            "Collecting pathvalidate<3,>=2.3.0\n",
            "  Downloading pathvalidate-2.4.1-py3-none-any.whl (18 kB)\n",
            "Collecting tcolorpy<1,>=0.0.5\n",
            "  Downloading tcolorpy-0.1.1-py3-none-any.whl (8.0 kB)\n",
            "Collecting mbstrdecoder<2,>=1.0.0\n",
            "  Downloading mbstrdecoder-1.0.1-py3-none-any.whl (7.8 kB)\n",
            "Collecting typepy[datetime]<2,>=1.1.1\n",
            "  Downloading typepy-1.2.0-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (21.0)\n",
            "Requirement already satisfied: importlib-metadata>=0.12 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (4.6.1)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (21.2.0)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.1.1)\n",
            "Collecting pluggy<1.0.0a1,>=0.12\n",
            "  Downloading pluggy-0.13.1-py2.py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: py>=1.8.2 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.10.0)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.3.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting huggingface-hub==0.0.12\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 68.6 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 67.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb~=0.10.22->-r mesh-transformer-jax/requirements.txt (line 5)) (1.15.0)\n",
            "Collecting subprocess32>=3.5.3\n",
            "  Downloading subprocess32-3.5.4.tar.gz (97 kB)\n",
            "\u001b[K     |████████████████████████████████| 97 kB 7.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb~=0.10.22->-r mesh-transformer-jax/requirements.txt (line 5)) (5.4.8)\n",
            "Collecting configparser>=3.8.1\n",
            "  Downloading configparser-5.0.2-py3-none-any.whl (19 kB)\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.1-py3-none-any.whl (7.5 kB)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb~=0.10.22->-r mesh-transformer-jax/requirements.txt (line 5)) (2.3)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.18-py3-none-any.whl (170 kB)\n",
            "\u001b[K     |████████████████████████████████| 170 kB 70.5 MB/s \n",
            "\u001b[?25hCollecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb~=0.10.22->-r mesh-transformer-jax/requirements.txt (line 5)) (2.8.1)\n",
            "Collecting sentry-sdk>=0.4.0\n",
            "  Downloading sentry_sdk-1.3.0-py2.py3-none-any.whl (133 kB)\n",
            "\u001b[K     |████████████████████████████████| 133 kB 59.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->-r mesh-transformer-jax/requirements.txt (line 7)) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->-r mesh-transformer-jax/requirements.txt (line 7)) (2021.5.30)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->-r mesh-transformer-jax/requirements.txt (line 7)) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->-r mesh-transformer-jax/requirements.txt (line 7)) (2.10)\n",
            "Collecting invoke<2.0,>=1.3\n",
            "  Downloading invoke-1.6.0-py3-none-any.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 64.1 MB/s \n",
            "\u001b[?25hCollecting paramiko>=2.4\n",
            "  Downloading paramiko-2.7.2-py2.py3-none-any.whl (206 kB)\n",
            "\u001b[K     |████████████████████████████████| 206 kB 71.9 MB/s \n",
            "\u001b[?25hCollecting pathlib2\n",
            "  Downloading pathlib2-2.3.6-py2.py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: opt-einsum in /usr/local/lib/python3.7/dist-packages (from jax~=0.2.12->-r mesh-transformer-jax/requirements.txt (line 13)) (3.3.0)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from Flask~=1.1.2->-r mesh-transformer-jax/requirements.txt (line 14)) (1.0.1)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from Flask~=1.1.2->-r mesh-transformer-jax/requirements.txt (line 14)) (1.1.0)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from Flask~=1.1.2->-r mesh-transformer-jax/requirements.txt (line 14)) (2.11.3)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.12)\n",
            "Requirement already satisfied: keras-nightly~=2.5.0.dev in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (2.5.0.dev2021032900)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (3.1.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.12.1)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.6.3)\n",
            "Requirement already satisfied: tensorboard~=2.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (2.5.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (0.2.0)\n",
            "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (2.5.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.1.2)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (0.36.2)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (0.4.0)\n",
            "Collecting google-cloud-core<2.0dev,>=1.4.1\n",
            "  Downloading google_cloud_core-1.7.1-py2.py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: google-auth<2.0dev,>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (1.32.1)\n",
            "Collecting google-resumable-media<2.0dev,>=1.2.0\n",
            "  Downloading google_resumable_media-1.3.1-py2.py3-none-any.whl (75 kB)\n",
            "\u001b[K     |████████████████████████████████| 75 kB 5.1 MB/s \n",
            "\u001b[?25hCollecting rehash\n",
            "  Downloading rehash-1.0.0-py2.py3-none-any.whl (8.7 kB)\n",
            "Requirement already satisfied: toolz>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from chex>=0.0.4->optax==0.0.6->-r mesh-transformer-jax/requirements.txt (line 9)) (0.11.1)\n",
            "Requirement already satisfied: dm-tree>=0.1.5 in /usr/local/lib/python3.7/dist-packages (from chex>=0.0.4->optax==0.0.6->-r mesh-transformer-jax/requirements.txt (line 9)) (0.1.6)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 71.8 MB/s \n",
            "\u001b[?25hCollecting fsspec>=2021.05.0\n",
            "  Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)\n",
            "\u001b[K     |████████████████████████████████| 118 kB 80.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (3.0.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (0.70.12.2)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (0.3.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.1.5)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.7-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 1.7 MB/s \n",
            "\u001b[?25hCollecting smmap<5,>=3.0.1\n",
            "  Downloading smmap-4.0.0-py2.py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.11.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.11.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.11.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (4.7.2)\n",
            "Requirement already satisfied: google-api-core<2.0.0dev,>=1.21.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (1.26.3)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (1.53.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (2018.9)\n",
            "Collecting google-crc32c<2.0dev,>=1.0\n",
            "  Downloading google_crc32c-1.1.2-cp37-cp37m-manylinux2014_x86_64.whl (38 kB)\n",
            "Requirement already satisfied: cffi>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from google-crc32c<2.0dev,>=1.0->google-resumable-media<2.0dev,>=1.2.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (1.14.6)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0->google-resumable-media<2.0dev,>=1.2.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (2.20)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.5.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.12->pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (3.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from jaxlib>=0.1.37->optax==0.0.6->-r mesh-transformer-jax/requirements.txt (line 9)) (1.4.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->Flask~=1.1.2->-r mesh-transformer-jax/requirements.txt (line 14)) (2.0.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->pytest==6.2.3->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (2.4.7)\n",
            "Collecting bcrypt>=3.1.3\n",
            "  Downloading bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 2.5 MB/s \n",
            "\u001b[?25hCollecting cryptography>=2.5\n",
            "  Downloading cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl (3.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2 MB 69.7 MB/s \n",
            "\u001b[?25hCollecting pynacl>=1.0.1\n",
            "  Downloading PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl (961 kB)\n",
            "\u001b[K     |████████████████████████████████| 961 kB 70.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.11.0->google-cloud-storage~=1.36.2->-r mesh-transformer-jax/requirements.txt (line 17)) (0.4.8)\n",
            "Collecting threadpoolctl>=2.0.0\n",
            "  Downloading threadpoolctl-2.2.0-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.24.1->lm-eval-harness==0.0.1->-r mesh-transformer-jax/requirements.txt (line 11)) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (0.4.4)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.8.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (0.6.1)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (1.3.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow-cpu~=2.5.0->-r mesh-transformer-jax/requirements.txt (line 16)) (3.1.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->-r mesh-transformer-jax/requirements.txt (line 20)) (0.2.5)\n",
            "Collecting starlette==0.14.2\n",
            "  Downloading starlette-0.14.2-py3-none-any.whl (60 kB)\n",
            "\u001b[K     |████████████████████████████████| 60 kB 7.7 MB/s \n",
            "\u001b[?25hCollecting h11>=0.8\n",
            "  Downloading h11-0.12.0-py3-none-any.whl (54 kB)\n",
            "\u001b[K     |████████████████████████████████| 54 kB 3.1 MB/s \n",
            "\u001b[?25hCollecting asgiref>=3.3.4\n",
            "  Downloading asgiref-3.4.1-py3-none-any.whl (25 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)\n",
            "\u001b[K     |████████████████████████████████| 294 kB 79.0 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl (142 kB)\n",
            "\u001b[K     |████████████████████████████████| 142 kB 82.5 MB/s \n",
            "\u001b[?25hCollecting async-timeout<4.0,>=3.0\n",
            "  Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n",
            "Collecting hiredis\n",
            "  Downloading hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (85 kB)\n",
            "\u001b[K     |████████████████████████████████| 85 kB 4.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.7/dist-packages (from gpustat->ray==1.4.1->-r mesh-transformer-jax/requirements.txt (line 12)) (7.352.0)\n",
            "Collecting blessings>=1.6\n",
            "  Downloading blessings-1.7-py3-none-any.whl (18 kB)\n",
            "Collecting opencensus-context==0.1.2\n",
            "  Downloading opencensus_context-0.1.2-py2.py3-none-any.whl (4.4 kB)\n",
            "Building wheels for collected packages: dm-haiku, lm-eval-harness, black, openai, pycountry, sqlitedict, subprocess32, func-timeout, ftfy, gpustat, pathtools\n",
            "  Building wheel for dm-haiku (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for dm-haiku: filename=dm_haiku-0.0.5.dev0-py3-none-any.whl size=531899 sha256=358a7ff789e81ceba8603848668da1f7aed725c0c3d4db14a2c488d09cb533a9\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-mar9uufw/wheels/06/28/69/ebaac5b2435641427299f29d88d005fb4e2627f4a108f0bdbc\n",
            "  Building wheel for lm-eval-harness (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lm-eval-harness: filename=lm_eval_harness-0.0.1-py3-none-any.whl size=100852 sha256=59990ab6df6a2895a043dbbabde852f752e35c50deffe96dfdbeb07075720868\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-mar9uufw/wheels/35/22/ec/71da337686fde904ec27f675d1bf73c4f253c48a6174d822b0\n",
            "  Building wheel for black (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for black: filename=black-20.8b1-py3-none-any.whl size=124195 sha256=b672672a10e5989dd0b26031f3a1dd5bf95ff6857c0b3e1f45526e42f965371c\n",
            "  Stored in directory: /root/.cache/pip/wheels/c5/85/79/f3af8daaf8037c0bf14beb3b7a1511a39b6e6902ca2aaf494e\n",
            "  Building wheel for openai (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for openai: filename=openai-0.6.4-py3-none-any.whl size=172215 sha256=09a03b80d55f149aad87686c8fc3b46e9ae93426452ca05c01f5707c981d245f\n",
            "  Stored in directory: /root/.cache/pip/wheels/54/c4/02/aa519fe2aaf97a9bba197a8585182c8c07802760351afdb64a\n",
            "  Building wheel for pycountry (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pycountry: filename=pycountry-20.7.3-py2.py3-none-any.whl size=10746883 sha256=a1c22a23fe1af147d7d4e5e80e94c419ba70912351faaf8855992695b3942e4b\n",
            "  Stored in directory: /root/.cache/pip/wheels/57/e8/3f/120ccc1ff7541c108bc5d656e2a14c39da0d824653b62284c6\n",
            "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sqlitedict: filename=sqlitedict-1.6.0-py3-none-any.whl size=14712 sha256=543aedfe025fd48a2f80d37c5aaeac63b85edcce11707385071b182ef10f50e3\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/dd/2c/0a57aadf6a7f26bec0af66d742c50af74d11967780f0bb7a7d\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-py3-none-any.whl size=6502 sha256=30d41b75da0febe371cec78516a811e546ce96b663d827c5454264513e18716d\n",
            "  Stored in directory: /root/.cache/pip/wheels/50/ca/fa/8fca8d246e64f19488d07567547ddec8eb084e8c0d7a59226a\n",
            "  Building wheel for func-timeout (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for func-timeout: filename=func_timeout-4.3.5-py3-none-any.whl size=15096 sha256=e565d15dde6d4b80489331c1854104341ac182133d5677856e41ba9594f7b5de\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/b5/a5/67c4364c354e141f5a1bd3ec568126f77877ab7554cf5af8cb\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ftfy: filename=ftfy-6.0.3-py3-none-any.whl size=41934 sha256=69c7067e309e434f634d33c3df68fa784d5f366712bfbdec12e8d46da3aeb1e6\n",
            "  Stored in directory: /root/.cache/pip/wheels/19/f5/38/273eb3b5e76dfd850619312f693716ac4518b498f5ffb6f56d\n",
            "  Building wheel for gpustat (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-0.6.0-py3-none-any.whl size=12617 sha256=b06416328b1988033903530e4b1b0403ae6c35d3dfeba75f2737ddefc823c0d1\n",
            "  Stored in directory: /root/.cache/pip/wheels/e6/67/af/f1ad15974b8fd95f59a63dbf854483ebe5c7a46a93930798b8\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=d7348d5e835e35698c8df7085c63ea3fe9f90d47bd81091b7f5e920096f03f23\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built dm-haiku lm-eval-harness black openai pycountry sqlitedict subprocess32 func-timeout ftfy gpustat pathtools\n",
            "Installing collected packages: mbstrdecoder, typepy, setuptools, requests, multidict, yarl, tqdm, smmap, regex, google-crc32c, DataProperty, async-timeout, zstandard, xxhash, ujson, typed-ast, tokenizers, threadpoolctl, tcolorpy, tabledata, sacremoses, rehash, pynacl, portalocker, pluggy, pathvalidate, pathspec, opencensus-context, mypy-extensions, msgfy, jsonlines, huggingface-hub, hiredis, google-resumable-media, google-cloud-core, gitdb, fsspec, cryptography, colorama, blessings, bcrypt, aiohttp, transformers, tqdm-multiprocess, subprocess32, starlette, sqlitedict, shortuuid, sentry-sdk, scikit-learn, sacrebleu, redis, pytest, pytablewriter, pydantic, pycountry, pybind11, py-spy, pathtools, pathlib2, paramiko, opencensus, openai, numexpr, mock, lm-dataformat, jmp, invoke, h11, gpustat, google-cloud-storage, GitPython, docker-pycreds, datasets, configparser, chex, black, best-download, asgiref, aioredis, aiohttp-cors, wandb, uvicorn, tensorflow-cpu, ray, optax, lm-eval-harness, func-timeout, ftfy, fastapi, fabric, einops, dm-haiku\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.2.0\n",
            "    Uninstalling setuptools-57.2.0:\n",
            "      Successfully uninstalled setuptools-57.2.0\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "  Attempting uninstall: regex\n",
            "    Found existing installation: regex 2019.12.20\n",
            "    Uninstalling regex-2019.12.20:\n",
            "      Successfully uninstalled regex-2019.12.20\n",
            "  Attempting uninstall: pluggy\n",
            "    Found existing installation: pluggy 0.7.1\n",
            "    Uninstalling pluggy-0.7.1:\n",
            "      Successfully uninstalled pluggy-0.7.1\n",
            "  Attempting uninstall: google-resumable-media\n",
            "    Found existing installation: google-resumable-media 0.4.1\n",
            "    Uninstalling google-resumable-media-0.4.1:\n",
            "      Successfully uninstalled google-resumable-media-0.4.1\n",
            "  Attempting uninstall: google-cloud-core\n",
            "    Found existing installation: google-cloud-core 1.0.3\n",
            "    Uninstalling google-cloud-core-1.0.3:\n",
            "      Successfully uninstalled google-cloud-core-1.0.3\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "  Attempting uninstall: pytest\n",
            "    Found existing installation: pytest 3.6.4\n",
            "    Uninstalling pytest-3.6.4:\n",
            "      Successfully uninstalled pytest-3.6.4\n",
            "  Attempting uninstall: numexpr\n",
            "    Found existing installation: numexpr 2.7.3\n",
            "    Uninstalling numexpr-2.7.3:\n",
            "      Successfully uninstalled numexpr-2.7.3\n",
            "  Attempting uninstall: google-cloud-storage\n",
            "    Found existing installation: google-cloud-storage 1.18.1\n",
            "    Uninstalling google-cloud-storage-1.18.1:\n",
            "      Successfully uninstalled google-cloud-storage-1.18.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.25.1 which is incompatible.\n",
            "google-cloud-bigquery 1.21.0 requires google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1, but you have google-resumable-media 1.3.1 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed DataProperty-0.52.0 GitPython-3.1.18 aiohttp-3.7.4.post0 aiohttp-cors-0.7.0 aioredis-1.3.1 asgiref-3.4.1 async-timeout-3.0.1 bcrypt-3.2.0 best-download-0.0.7 black-20.8b1 blessings-1.7 chex-0.0.8 colorama-0.4.4 configparser-5.0.2 cryptography-3.4.7 datasets-1.10.2 dm-haiku-0.0.5.dev0 docker-pycreds-0.4.0 einops-0.3.0 fabric-2.6.0 fastapi-0.67.0 fsspec-2021.7.0 ftfy-6.0.3 func-timeout-4.3.5 gitdb-4.0.7 google-cloud-core-1.7.1 google-cloud-storage-1.36.2 google-crc32c-1.1.2 google-resumable-media-1.3.1 gpustat-0.6.0 h11-0.12.0 hiredis-2.0.0 huggingface-hub-0.0.12 invoke-1.6.0 jmp-0.0.2 jsonlines-2.0.0 lm-dataformat-0.0.19 lm-eval-harness-0.0.1 mbstrdecoder-1.0.1 mock-4.0.3 msgfy-0.1.0 multidict-5.1.0 mypy-extensions-0.4.3 numexpr-2.7.2 openai-0.6.4 opencensus-0.7.13 opencensus-context-0.1.2 optax-0.0.6 paramiko-2.7.2 pathlib2-2.3.6 pathspec-0.9.0 pathtools-0.1.2 pathvalidate-2.4.1 pluggy-0.13.1 portalocker-2.3.0 py-spy-0.3.7 pybind11-2.6.2 pycountry-20.7.3 pydantic-1.8.2 pynacl-1.4.0 pytablewriter-0.58.0 pytest-6.2.3 ray-1.4.1 redis-3.5.3 regex-2021.7.6 rehash-1.0.0 requests-2.25.1 sacrebleu-1.5.0 sacremoses-0.0.45 scikit-learn-0.24.2 sentry-sdk-1.3.0 setuptools-51.3.3 shortuuid-1.0.1 smmap-4.0.0 sqlitedict-1.6.0 starlette-0.14.2 subprocess32-3.5.4 tabledata-1.2.0 tcolorpy-0.1.1 tensorflow-cpu-2.5.0 threadpoolctl-2.2.0 tokenizers-0.10.3 tqdm-4.45.0 tqdm-multiprocess-0.0.11 transformers-4.8.2 typed-ast-1.4.3 typepy-1.2.0 ujson-4.0.2 uvicorn-0.14.0 wandb-0.10.33 xxhash-2.0.2 yarl-1.6.3 zstandard-0.15.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "google",
                  "pkg_resources"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Processing ./mesh-transformer-jax\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "Collecting jax==0.2.12\n",
            "  Downloading jax-0.2.12.tar.gz (590 kB)\n",
            "\u001b[K     |████████████████████████████████| 590 kB 5.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.12 in /usr/local/lib/python3.7/dist-packages (from jax==0.2.12) (1.19.5)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from jax==0.2.12) (0.12.0)\n",
            "Requirement already satisfied: opt_einsum in /usr/local/lib/python3.7/dist-packages (from jax==0.2.12) (3.3.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py->jax==0.2.12) (1.15.0)\n",
            "Building wheels for collected packages: mesh-transformer, jax\n",
            "  Building wheel for mesh-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mesh-transformer: filename=mesh_transformer-0.0.0-py3-none-any.whl size=24001 sha256=49e9ee9d0c301e4f06d874b96bee77a4b54599f7af2156f6336b609f1698e67f\n",
            "  Stored in directory: /root/.cache/pip/wheels/56/bd/89/b1f6b2f3d6b938d0c5812ee97756a1afd32521bea293543863\n",
            "  Building wheel for jax (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jax: filename=jax-0.2.12-py3-none-any.whl size=682484 sha256=f1c4bcc252669970d7e8b39a34755b9545c7121afb4e7b68523b51f3d1aa2fcb\n",
            "  Stored in directory: /root/.cache/pip/wheels/f6/4d/e5/73eec5070b77f25664c67bd793d4eb97f41bbf9be7afafd15e\n",
            "Successfully built mesh-transformer jax\n",
            "Installing collected packages: mesh-transformer, jax\n",
            "  Attempting uninstall: jax\n",
            "    Found existing installation: jax 0.2.17\n",
            "    Uninstalling jax-0.2.17:\n",
            "      Successfully uninstalled jax-0.2.17\n",
            "Successfully installed jax-0.2.12 mesh-transformer-0.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aO1UXepF-0Uq"
      },
      "source": [
        "## Setup Model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ex0qJgaueZtJ"
      },
      "source": [
        "import os\n",
        "import requests \n",
        "from jax.config import config\n",
        "\n",
        "colab_tpu_addr = os.environ['COLAB_TPU_ADDR'].split(':')[0]\n",
        "url = f'http://{colab_tpu_addr}:8475/requestversion/tpu_driver0.1_dev20210607'\n",
        "requests.post(url)\n",
        "\n",
        "# The following is required to use TPU Driver as JAX's backend.\n",
        "config.FLAGS.jax_xla_backend = \"tpu_driver\"\n",
        "config.FLAGS.jax_backend_target = \"grpc://\" + os.environ['COLAB_TPU_ADDR']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NIgUVdFLe4A8"
      },
      "source": [
        "Sometimes the next step errors for some reason, just run it again ¯\\\\\\_(ツ)\\_/¯"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-A5IGYSaeze3"
      },
      "source": [
        "import time\n",
        "\n",
        "import jax\n",
        "from jax.experimental import maps\n",
        "import numpy as np\n",
        "import optax\n",
        "import transformers\n",
        "\n",
        "from mesh_transformer.checkpoint import read_ckpt\n",
        "from mesh_transformer.sampling import nucleaus_sample\n",
        "from mesh_transformer.transformer_shard import CausalTransformer\n",
        "\n",
        "from tqdm import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QAgKq-X2kmba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "fbf4b2460d7d4dad804e7e727b6b4c59",
            "ab2afcba1de54204a450f8d780f7cd9d",
            "f97e3f83eec64b9da6426a1663d66b69",
            "bb49011c43f64924adaa20aaec136aee",
            "cb828710850741b58ed357ef2551179c",
            "3a7a31a0d1f34e329b0004388fa3e653",
            "3c24c5146ed84ad6a13cecac908d77b2",
            "59154f40ef8846dda76016a0c6496343",
            "fc508e48c9754c9c86328d336f8b8a38",
            "b4cd19fb39354e0586737b2323ef7470",
            "0678f4d1926a4a73915bc5a86e031e37",
            "b08f63d6ea6642b5af67f1f7ffc23bbe",
            "5fc5e9c458f4469e9f2b5b3b1f92763c",
            "aea7b69157614da791d66b3d14bbfab6",
            "d49ace7f94e5493c8c266951df508bcd",
            "d0ec6249898e46b4ae5085a1d60cb6d2",
            "f085f95f4f2b41d39ccadd41d51563c7",
            "88d146ac80b441b8b86e534419d852c9",
            "f1496cdd9a3e432d8a03d82ad47f275d",
            "bbd1da114e054943a0e90ca00a522e14",
            "9e815b652ed443e7b0a21f66282b4db9",
            "53df321d7fbc4a959913d15548bbb577",
            "f65669d413a84b52a99ea47ffd10df86",
            "7649a02849bd4bcebf30a57ac2afe4d3"
          ]
        },
        "outputId": "396d5943-2263-4a0a-d326-ee45ca260bec"
      },
      "source": [
        "params = {\n",
        "  \"layers\": 28,\n",
        "  \"d_model\": 4096,\n",
        "  \"n_heads\": 16,\n",
        "  \"n_vocab\": 50400,\n",
        "  \"norm\": \"layernorm\",\n",
        "  \"pe\": \"rotary\",\n",
        "  \"pe_rotary_dims\": 64,\n",
        "\n",
        "  \"seq\": 2048,\n",
        "  \"cores_per_replica\": 8,\n",
        "  \"per_replica_batch\": 1,\n",
        "}\n",
        "\n",
        "per_replica_batch = params[\"per_replica_batch\"]\n",
        "cores_per_replica = params[\"cores_per_replica\"]\n",
        "seq = params[\"seq\"]\n",
        "\n",
        "\n",
        "params[\"sampler\"] = nucleaus_sample\n",
        "\n",
        "# here we \"remove\" the optimizer parameters from the model (as we don't need them for inference)\n",
        "params[\"optimizer\"] = optax.scale(0)\n",
        "\n",
        "mesh_shape = (jax.device_count() // cores_per_replica, cores_per_replica)\n",
        "devices = np.array(jax.devices()).reshape(mesh_shape)\n",
        "\n",
        "maps.thread_resources.env = maps.ResourceEnv(maps.Mesh(devices, ('dp', 'mp')))\n",
        "\n",
        "tokenizer = transformers.GPT2TokenizerFast.from_pretrained('gpt2')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fbf4b2460d7d4dad804e7e727b6b4c59",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1042301.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc508e48c9754c9c86328d336f8b8a38",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f085f95f4f2b41d39ccadd41d51563c7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1355256.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yFgRkUgfiNdA"
      },
      "source": [
        "Here we create the network and load the parameters from the downloaded files. Expect this to take around 5 minutes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lwNETD2Uk8nu",
        "outputId": "d27f59b7-34ed-4ea6-9f4d-5db63abe796d"
      },
      "source": [
        "total_batch = per_replica_batch * jax.device_count() // cores_per_replica\n",
        "\n",
        "network = CausalTransformer(params)\n",
        "\n",
        "network.state = read_ckpt(network.state, \"step_383500/\", devices.shape[1])\n",
        "\n",
        "network.state = network.move_xmap(network.state, np.zeros(cores_per_replica))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/jax/experimental/maps.py:412: UserWarning: xmap is an experimental feature and probably has bugs!\n",
            "  warn(\"xmap is an experimental feature and probably has bugs!\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "key shape (8, 2)\n",
            "in shape (1, 2048)\n",
            "dp 1\n",
            "mp 8\n",
            "Total parameters: 6053381344\n",
            "read from disk/gcs in 43.234s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-eT7Sw6if4J"
      },
      "source": [
        "## Run Model\n",
        "\n",
        "Finally, we are ready to infer with the model! The first sample takes around a minute due to compilation, but after that it should only take about 10 seconds per sample.\n",
        "\n",
        "Feel free to mess with the different sampling parameters (top_p and temp), as well as the length of the generations (gen_len, causes a recompile when changed).\n",
        "\n",
        "You can also change other things like per_replica_batch in the previous cells to change how many generations are done in parallel. A larger batch has higher latency but higher throughput when measured in tokens generated/s. This is useful for doing things like best-of-n cherry picking.\n",
        "\n",
        "*Tip for best results: Make sure your prompt does not have any trailing spaces, which tend to confuse the model due to the BPE tokenization used during training.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V71QL9BgRFF7"
      },
      "source": [
        "# allow text wrapping in generated output: https://stackoverflow.com/a/61401455\n",
        "from IPython.display import HTML, display\n",
        "\n",
        "def set_css():\n",
        "  display(HTML('''\n",
        "  <style>\n",
        "    pre {\n",
        "        white-space: pre-wrap;\n",
        "    }\n",
        "  </style>\n",
        "  '''))\n",
        "get_ipython().events.register('pre_run_cell', set_css)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "ZVzs2TYlvYeX",
        "outputId": "c0a3d052-00f5-4783-8e68-a40dce84af87"
      },
      "source": [
        "def infer(context, top_p=0.9, temp=1.0, gen_len=512):\n",
        "    tokens = tokenizer.encode(context)\n",
        "\n",
        "    provided_ctx = len(tokens)\n",
        "    pad_amount = seq - provided_ctx\n",
        "\n",
        "    padded_tokens = np.pad(tokens, ((pad_amount, 0),)).astype(np.uint32)\n",
        "    batched_tokens = np.array([padded_tokens] * total_batch)\n",
        "    length = np.ones(total_batch, dtype=np.uint32) * len(tokens)\n",
        "\n",
        "    start = time.time()\n",
        "    output = network.generate(batched_tokens, length, gen_len, {\"top_p\": np.ones(total_batch) * top_p, \"temp\": np.ones(total_batch) * temp})\n",
        "\n",
        "    samples = []\n",
        "    decoded_tokens = output[1][0]\n",
        "\n",
        "    for o in decoded_tokens[:, :, 0]:\n",
        "      samples.append(f\"\\033[1m{context}\\033[0m{tokenizer.decode(o)}\")\n",
        "\n",
        "    # print(f\"completion done in {time.time() - start:06}s\")\n",
        "    return samples\n",
        "\n",
        "# print(infer(\"EleutherAI is\")[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2QDDvNTECqFY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "8f6f20e3-e367-4efd-e745-9ae51fe6147c"
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(\"gpt3-test-july20.csv\", sep=\"\\t\")\n",
        "df.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cancel my account</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i want to add new line</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>speak with an agent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>am i able to cancel my plan and close ts numbe...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>unfortunately i am moving somewhere without fr...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                             message\n",
              "0                                  cancel my account\n",
              "1                             i want to add new line\n",
              "2                                speak with an agent\n",
              "3  am i able to cancel my plan and close ts numbe...\n",
              "4  unfortunately i am moving somewhere without fr..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "hS2na9LkUymJ",
        "outputId": "09a06b82-94f1-4cf3-bfa6-feffbf740463"
      },
      "source": [
        "prompt = \"\"\"\n",
        "classify the following text\n",
        "text: i need help.\n",
        "class: help generic\n",
        "text: need assistance with my billing.\n",
        "class: billing issue\n",
        "text: help logging into my freedom account.\n",
        "class: account login issue\n",
        "text: can you help me i got locked out of my iphone.\n",
        "class: unlock phone\n",
        "text: want to speak to a representative to help me please.\n",
        "class: speak agent\n",
        "text: help with my freedom network.\n",
        "class: network issue\n",
        "text: need to change my current address.\n",
        "class: change address\n",
        "text: i need a person now.\n",
        "class: speak agent\n",
        "text: unlock my huawei device.\n",
        "class: unlock phone\n",
        "text: help with billing.\n",
        "class: billing issue\n",
        "text: check my account number.\n",
        "class: account issue\n",
        "text: we might need to change s plan because he's incurring overage charges\n",
        "class: change plan\n",
        "text: i will be moving and need to change the address on my account\n",
        "class: change address\n",
        "text: help logging into my freedom account.\n",
        "class: account login issue\n",
        "text: \"\"\"\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "hduqFQ1oFFRF",
        "outputId": "43fae670-9560-4c6f-e478-f158c54aaf00"
      },
      "source": [
        "input = prompt+\"so that's awesome and all i needed\"+\".\"\n",
        "# print(input)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4occHYlVU5Ox",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 752
        },
        "outputId": "900a8552-3146-4b0b-e8c5-b96c4273c544"
      },
      "source": [
        "out = infer(input, top_p=0.4, temp=0., gen_len = 64)[0]\n",
        "print(out)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1m\n",
            "classify the following text\n",
            "text: i need help.\n",
            "class: help generic\n",
            "text: need assistance with my billing.\n",
            "class: billing issue\n",
            "text: help logging into my freedom account.\n",
            "class: account login issue\n",
            "text: can you help me i got locked out of my iphone.\n",
            "class: unlock phone\n",
            "text: want to speak to a representative to help me please.\n",
            "class: speak agent\n",
            "text: help with my freedom network.\n",
            "class: network issue\n",
            "text: need to change my current address.\n",
            "class: change address\n",
            "text: i need a person now.\n",
            "class: speak agent\n",
            "text: unlock my huawei device.\n",
            "class: unlock phone\n",
            "text: help with billing.\n",
            "class: billing issue\n",
            "text: check my account number.\n",
            "class: account issue\n",
            "text: we might need to change s plan because he's incurring overage charges\n",
            "class: change plan\n",
            "text: i will be moving and need to change the address on my account\n",
            "class: change address\n",
            "text: help logging into my freedom account.\n",
            "class: account login issue\n",
            "\n",
            "text: so that's awesome and all i needed.\u001b[0m\n",
            "class: help generic\n",
            "text: can you help me i got locked out of my iphone.\n",
            "class: unlock phone\n",
            "text: want to speak to a representative to help me please.\n",
            "class: speak agent\n",
            "text: help with my freedom network.\n",
            "class: network issue\n",
            "text: need\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "wAl4H5KfGWn8",
        "outputId": "21e99e89-3109-4250-b60a-ec34f040ec09"
      },
      "source": [
        "out.split(\"\\x1b[0m\")[1].split(\"\\n\")[1].split(\":\")[1].strip()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'help generic'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Luj1GqR-Gcj8",
        "outputId": "e7d59d5d-4b3e-4188-91a2-d1e98e98d4ce"
      },
      "source": [
        "data = []\n",
        "for msg in tqdm(list(df.message)):\n",
        "  input = prompt+msg+\".\"\n",
        "  out = infer(input, top_p=0.5, temp=0., gen_len = 64)[0]\n",
        "  try:\n",
        "    c = out.split(\"\\x1b[0m\")[1].split(\"\\n\")[1].split(\":\")[1]\n",
        "  except:\n",
        "    print(\"error with the following message\")\n",
        "    print(out)\n",
        "    c = \"\"\n",
        "  data.append(\n",
        "      {\n",
        "          \"text\": msg,\n",
        "          \"class\": c.strip()\n",
        "      }\n",
        "  )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 118/118 [03:48<00:00,  1.94s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "GuGt7iITHASc",
        "outputId": "4061d49c-5b8f-4128-ad61-5be76c7fa0a3"
      },
      "source": [
        "df_res = pd.DataFrame(data=data)\n",
        "df_res"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cancel my account</td>\n",
              "      <td>cancel account</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i want to add new line</td>\n",
              "      <td>add new line</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>speak with an agent</td>\n",
              "      <td>speak agent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>am i able to cancel my plan and close ts numbe...</td>\n",
              "      <td>cancel plan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>unfortunately i am moving somewhere without fr...</td>\n",
              "      <td>help</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>113</th>\n",
              "      <td>i will cancel close my account and pay the rem...</td>\n",
              "      <td>cancel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>114</th>\n",
              "      <td>i would like to close my account</td>\n",
              "      <td>close account</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>115</th>\n",
              "      <td>i just want to close my account</td>\n",
              "      <td>close account</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>116</th>\n",
              "      <td>so i have three prepaid numbers that i paid ye...</td>\n",
              "      <td>add a new number</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>117</th>\n",
              "      <td>i basically want to keep the same plan that th...</td>\n",
              "      <td>change plan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>118 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                  text             class\n",
              "0                                    cancel my account    cancel account\n",
              "1                               i want to add new line      add new line\n",
              "2                                  speak with an agent       speak agent\n",
              "3    am i able to cancel my plan and close ts numbe...       cancel plan\n",
              "4    unfortunately i am moving somewhere without fr...              help\n",
              "..                                                 ...               ...\n",
              "113  i will cancel close my account and pay the rem...            cancel\n",
              "114                   i would like to close my account     close account\n",
              "115                    i just want to close my account     close account\n",
              "116  so i have three prepaid numbers that i paid ye...  add a new number\n",
              "117  i basically want to keep the same plan that th...       change plan\n",
              "\n",
              "[118 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "j1DWqdx5UR5Z",
        "outputId": "2c2fc8cf-5a0d-44df-9473-b30f47ac4050"
      },
      "source": [
        "len(df_res[\"class\"].unique())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "jy0NsWRsHrNV",
        "outputId": "c86cbbf0-d03c-4841-e117-2d03a03b8bd6"
      },
      "source": [
        "# df_res.to_csv(\"gpt3_group_7.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wcx5MiYXlRNn"
      },
      "source": [
        "context summarization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsKF6O3EH-i2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "c697c685-136a-44b4-e8bb-9689d96200cb"
      },
      "source": [
        "prompt = \"\"\"\n",
        "extract context from the following text\n",
        "\n",
        "text: need assistance with my billing.\n",
        "context: no context given\n",
        "text: i am unable to access it due to the fact that my email associated with the account is no longer able to be used\n",
        "context: email no longer used\n",
        "text: i will be moving and need to change the address on my account\n",
        "context: will be moving\n",
        "text: help logging into my freedom account.\n",
        "context: no context given\n",
        "text: we might need to change s plan because he's incurring overage charges\n",
        "context: incurring overage charges\n",
        "text: and received a message saying we need a stronger pin\n",
        "context: require stronger pin\n",
        "text: \"\"\"\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "GLZwR-1soCmq",
        "outputId": "83326054-70db-4b10-b42d-d9524630d808"
      },
      "source": [
        "contexts = []\n",
        "for msg in tqdm(list(df.message)):\n",
        "  input = prompt+msg+\".\"\n",
        "  out = infer(input, top_p=0.5, temp=0., gen_len = 64)[0]\n",
        "  try:\n",
        "    c = out.split(\"\\x1b[0m\")[1].split(\"\\n\")[1].split(\":\")[1]\n",
        "  except:\n",
        "    print(\"error with the following message\")\n",
        "    print(out)\n",
        "    c = \"\"\n",
        "\n",
        "  contexts.append(c.strip())\n",
        "\n",
        "df_res[\"context\"] = contexts"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            " 40%|███▉      | 47/118 [01:31<02:17,  1.94s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "error with the following message\n",
            "\u001b[1m\n",
            "extract context from the following text\n",
            "\n",
            "text: need assistance with my billing.\n",
            "context: no context given\n",
            "text: i am unable to access it due to the fact that my email associated with the account is no longer able to be used\n",
            "context: email no longer used\n",
            "text: i will be moving and need to change the address on my account\n",
            "context: will be moving\n",
            "text: help logging into my freedom account.\n",
            "context: no context given\n",
            "text: we might need to change s plan because he's incurring overage charges\n",
            "context: incurring overage charges\n",
            "text: and received a message saying we need a stronger pin\n",
            "context: require stronger pin\n",
            "text: i 'm on direct deposit to pay my bill and you have n't taken money from me yet for march.\u001b[0m\n",
            "context:n't taken money\n",
            "text: we are a part of a p&g.\n",
            "context: p&g\n",
            "text: we are a p&g, and the p&g is a p&g.\n",
            "context: p&g\n",
            "text: the p&g is a p&\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 118/118 [03:48<00:00,  1.94s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2-baqpUAoLA7",
        "outputId": "68cba1d6-13ac-4950-be14-c64c8577fb83"
      },
      "source": [
        "df_res.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>class</th>\n",
              "      <th>context</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cancel my account</td>\n",
              "      <td>cancel account</td>\n",
              "      <td>no context given</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i want to add new line</td>\n",
              "      <td>add new line</td>\n",
              "      <td>no context given</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>speak with an agent</td>\n",
              "      <td>speak agent</td>\n",
              "      <td>no context given</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>am i able to cancel my plan and close ts numbe...</td>\n",
              "      <td>cancel plan</td>\n",
              "      <td>not here</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>unfortunately i am moving somewhere without fr...</td>\n",
              "      <td>help</td>\n",
              "      <td>will be moving</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  ...           context\n",
              "0                                  cancel my account  ...  no context given\n",
              "1                             i want to add new line  ...  no context given\n",
              "2                                speak with an agent  ...  no context given\n",
              "3  am i able to cancel my plan and close ts numbe...  ...          not here\n",
              "4  unfortunately i am moving somewhere without fr...  ...    will be moving\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "-VA47GhqogRo",
        "outputId": "334132f1-9645-468e-cc51-0e0186851986"
      },
      "source": [
        "df_res.to_csv(\"gpt3-topic-mining-july-results.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yhg130ovolTx"
      },
      "source": [
        "\"\"\"\n",
        "text:\n",
        "content: ... \\n\n",
        "context:  ... \\n\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M_cvdmICubCF"
      },
      "source": [
        "# Topic clustering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tiZ_3nW8ufMC"
      },
      "source": [
        "!pip install sentence-transformers\n",
        "!pip install umap-learn\n",
        "!pip install hdbscan"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Mfin7fZuzG9"
      },
      "source": [
        "import pandas as pd\n",
        "from sentence_transformers import SentenceTransformer\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1WW5rtHsvBnq"
      },
      "source": [
        "df = pd.read_csv(\"gpt3-topic-mining-results.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 672,
          "referenced_widgets": [
            "ab283591978f478f90382624b50b5c99",
            "02a9b039d15640a4babc86068f5f3382",
            "404a8a74ddb74b668330e3ac109ad936",
            "f79535e1fe9042b0811e75c32f1103b3",
            "45ec2f2169f64bbaabc1da59af92010f",
            "c19f935f197548b182e5e2ec1a845b20",
            "7724d7e460f74d99bb8850dd19635155",
            "b955aa83ae574583b2164fd3dcbe8dc0",
            "d020c2a260604b3abce3ba0330ad82bc",
            "7ebf8dbff8204bac906791141d811e2f",
            "59840c160c2d4fe083287a1c7cf14f01",
            "d50d279e12d743c6b4230277c25962fa",
            "99f7eb3fd674409a88c11a62accfed6f",
            "54241adcb36a4c37b2b05a17aecd7380",
            "09a48d61d2c84fb689ce17be4debac38",
            "8e3f9c8ef9254155829342977206b42f",
            "f995731eb92f4121a920ed4c6622da86",
            "0a1c296f0fb843eda0a9ee41e07cafac",
            "7a559bac6132437782b3e92cfe80b1d2",
            "cac3a28737b94d64a5b37984bfab04b5",
            "c0e32f91a58641a1b54e7c93333e2fa9",
            "fe7ce0e260104613a68fedda4babe779",
            "914dd6d5ef89412e8f5938af87580f13",
            "fa89483651f1418d96bcaa530f9679bf",
            "8319f91b448748cfadcb5e308df88ea4",
            "11507eaa281c45069bd7d9aa212fa8a8",
            "bea259f564e8420994d3343a78bd5d92",
            "479ce3a697554b80825dc81d0911bb38",
            "f2196b351e4e4ee4939905c07c13fffe",
            "0711d4ad8190486389ed4b2395757e81",
            "395811ef31e3437d83485c8ad39265a6",
            "de22013fe820480e9a1d4dc09e4cc761",
            "da696617d67443f38865fa1d02990543",
            "29801af3d4fe4ccf9de84b98e35cb345",
            "49692a8a42404f658afa591fbdac10a8",
            "88b8e9ae983d4a2fa04fe183a5e08501",
            "0cb46b25cfff462995e3e81d1c4d907a",
            "2b8fb883c8be4cd28f17ff8e0cc14966",
            "7477d052a4b242a9ab452c5ab270fe2e",
            "537a37aa0e9d40c7a1716226d34dd278",
            "146a4f5514ab4d6f960873729fef517f",
            "849338533ab24feda2e5b672bbd0728d",
            "e7d7f054cfdd4d699b7c8b5ab6f7c432",
            "98d56955455a4618a117c8251504b141",
            "41df3185c5c14a148394847bde90575c",
            "b9218bbda6104eb49aabc2c3993848ea",
            "ef7f7be28b0948bbbba3ed52ca4fbb22",
            "6f166522c6f244548e0fc4412f1cf56d",
            "f72ad09c93df4e6fb6147a295bbecf58",
            "6b2f9adb62e44936baf4e5b7219bd77c",
            "ef86a8bf9da142689cbeed5ec2261b86",
            "61efd84f33334eb6815a41eb53f22914",
            "46e8b5c065f543cfb3add6839ce3d60f",
            "a9b8e9c8945f45eeaf33fe9e26d1dc76",
            "25b6eacdb5b3425dbd09e47da187b0c7",
            "85ff2ba5203b4a62a3154ab7cacf0c5f",
            "2a99debc45d54080ab32a03d8e27ac96",
            "715747079ead41daae87c9a6e45fbf5f",
            "19fc43e759504de6878953fef71fc4d1",
            "af612ad2dbcc48ce82a2fec347165787",
            "b53c7e66eab6421b864e1f4efdeffda3",
            "8ee100d28cab4898902823690809f3f9",
            "80e0af1fbe9c47f39bd6da7c54f0a133",
            "4d0dae66202d44fa8df3f9b17f0ea21f",
            "4c4fa8321fc0483bbb50a99da0064237",
            "07248052e1d543729b34fb3a4a4c4418",
            "1df916cccb7a4b55bebb3cbe86ee7f92",
            "ed62fe818f4b46c3a1ca02050b61780a",
            "25a8df472e0c4c9cae5655fad29a18c3",
            "9626f232541d4102bc25423df412f872",
            "ca6f4d24e07e44b8ad9907014389f680",
            "281c561552c547b0b98fee51ce346da0",
            "a1274fdf8a494752ab8b22d377d5223b",
            "7bd8131fa5fc40f7b3b60de6b8827039",
            "52b98a05ca9c43208721203c3b42f3cd",
            "ccb082b637104a97bd3a92d97dc2ad30",
            "5ee473403a6e402db6f1d507e51b3f7d",
            "18e91362094e4caeb248c18ade77faf4",
            "d266a10b4c16411b92a177d8ae06c905",
            "d2103d2a869640c0a5450b6d1a2f3612",
            "866b87e3ef8e40519f9474bda1475474",
            "a6bc1ee1fd1d4a0eb8c29aadf2dbf444",
            "e000b07793e74d7f8951dd98215b499c",
            "0b142d3e34894931ad4a12a755810e3e",
            "ef5ee2c450424860bb2e8067f46753fd",
            "3ba798647eea46dcb8a51e30764a934e",
            "af97567fc4204c129dfeb2fb593ed4fa",
            "7a2e95a96a594e72a69843eea0dee287",
            "61e2a230db48484e80de7f1728b5d4bc",
            "50138ae3a8b74213a6a25823c4f2c154",
            "c73a942bfca845d080974dcf5f5ae966",
            "693d8a5bf2ff40f7a017268fa2c0bbf4",
            "eca62b86260c4c379e11fac6dde84ac2",
            "2b4a4f3739c24a8e8062b6a2f825942b",
            "f5b20e506f5a4b438205e00259718199",
            "959df93b8b4a4c2fb6168543fbfe888a",
            "bc45d04ae8884f26a55774201f0942ff",
            "97eda0a864c24f589c316f5ad9e086c8",
            "8f15f816e5aa492d8e6a6f5bfe46334f",
            "479454e31211411b98e83a9e4494d325",
            "218f9b546c064621883e33f9dd9241ae",
            "e7bd0d57ed1c4c73866efea49333e5b1",
            "ad758c4162d64280b4e26d49b14a8cdd",
            "5e9d3995cd53456d82d0df551207f009"
          ]
        },
        "id": "Mn3sc5qKvHo0",
        "outputId": "882a17c5-0449-4437-8333-055bab659c48"
      },
      "source": [
        "# settings\n",
        "\"\"\"\n",
        "couple models to try\n",
        " paraphrase-distilroberta-base-v2 \n",
        " paraphrase-TinyBERT-L6-v2 \n",
        " stsb-distilroberta-base-v2\n",
        " \n",
        "\"\"\"\n",
        "pretrained_model = \"paraphrase-distilroberta-base-v2\"\n",
        "model = SentenceTransformer(pretrained_model)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab283591978f478f90382624b50b5c99",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=736.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d020c2a260604b3abce3ba0330ad82bc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=3721.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f995731eb92f4121a920ed4c6622da86",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=686.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8319f91b448748cfadcb5e308df88ea4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=122.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da696617d67443f38865fa1d02990543",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456356.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "146a4f5514ab4d6f960873729fef517f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=229.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f72ad09c93df4e6fb6147a295bbecf58",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=328515953.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2a99debc45d54080ab32a03d8e27ac96",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=53.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4c4fa8321fc0483bbb50a99da0064237",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=239.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a1274fdf8a494752ab8b22d377d5223b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1355881.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "866b87e3ef8e40519f9474bda1475474",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1123.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61e2a230db48484e80de7f1728b5d4bc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798293.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bc45d04ae8884f26a55774201f0942ff",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=190.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ch87XT-zvTat",
        "outputId": "1215b3b2-fc37-4608-ca4d-5b9ad5950ac7"
      },
      "source": [
        "topics = list(df[\"class\"])\n",
        "embeddings = model.encode(topics)\n",
        "print(embeddings.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(118, 768)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1J9JbBFKxEJA"
      },
      "source": [
        "## reduce dimensionality with umap"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0Sb1sx0wBop",
        "outputId": "888c90f1-5120-4e10-cd35-418ba38afd64"
      },
      "source": [
        "import umap\n",
        "umap_embeddings = umap.UMAP(n_neighbors=15, \n",
        "                            n_components=5, \n",
        "                            metric='cosine').fit_transform(embeddings)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/numba/np/ufunc/parallel.py:363: NumbaWarning: The TBB threading layer requires TBB version 2019.5 or later i.e., TBB_INTERFACE_VERSION >= 11005. Found TBB_INTERFACE_VERSION = 9107. The TBB threading layer is disabled.\n",
            "  warnings.warn(problem)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RYvNJX4FvmJM"
      },
      "source": [
        "## clustering - HDBSCAN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FFgnKB5vcLG"
      },
      "source": [
        "import hdbscan\n",
        "cluster_hdbscan = hdbscan.HDBSCAN(min_cluster_size=15,\n",
        "                          metric='euclidean',                      \n",
        "                          cluster_selection_method='eom').fit(embeddings)\n",
        "\n",
        "cluster_hdbscan_reduced = hdbscan.HDBSCAN(min_cluster_size=15,\n",
        "                          metric='euclidean',                      \n",
        "                          cluster_selection_method='eom').fit(umap_embeddings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_cuKW_vvweW"
      },
      "source": [
        "df[\"grouped_topics_hdbscan\"] = cluster_hdbscan.labels_\n",
        "df[\"grouped_topics_hdbscan_reduced\"] = cluster_hdbscan_reduced.labels_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xxyDDHQnv35U",
        "outputId": "e60a622e-6a7b-4c6a-91e0-08da48635c6f"
      },
      "source": [
        "cluster.labels_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 2,  0, -1,  2,  0, -1,  0,  1,  1, -1,  2,  2,  2,  1,  1,  0, -1,\n",
              "        1,  1, -1,  0,  0, -1, -1, -1,  2,  2,  1,  1,  0,  0,  2,  2,  2,\n",
              "        1,  1, -1, -1, -1,  2,  2,  2,  2, -1,  0,  1, -1, -1, -1, -1,  1,\n",
              "        1, -1,  2,  2,  1,  0,  0, -1,  2,  2,  0,  0,  1,  2, -1,  1,  1,\n",
              "       -1, -1,  2, -1,  2,  1,  2, -1,  1, -1,  1,  2,  2, -1,  0,  1, -1,\n",
              "       -1,  0,  2,  2,  2,  0,  1,  1,  1,  2, -1, -1,  1,  0,  1,  1,  1,\n",
              "        2, -1,  2, -1,  2,  1,  0,  2,  2,  2,  2,  2,  2,  2,  1, -1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lU4vC7jVv4X5",
        "outputId": "4a830347-d393-47c5-a2e2-8a36d46dcf4e"
      },
      "source": [
        "df.head(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>class</th>\n",
              "      <th>context</th>\n",
              "      <th>grouped_topics</th>\n",
              "      <th>grouped_topics_kmeans</th>\n",
              "      <th>grouped_topics_kmeans_reduced</th>\n",
              "      <th>grouped_topics_hdbscan</th>\n",
              "      <th>grouped_topics_hdbscan_reduced</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cancel my account</td>\n",
              "      <td>cancel account</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i want to add new line</td>\n",
              "      <td>add new line</td>\n",
              "      <td>no context given</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>speak with an agent</td>\n",
              "      <td>speak agent</td>\n",
              "      <td>no context given</td>\n",
              "      <td>-1</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>am i able to cancel my plan and close ts numbe...</td>\n",
              "      <td>cancel plan</td>\n",
              "      <td>not here</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>unfortunately i am moving somewhere without fr...</td>\n",
              "      <td>help</td>\n",
              "      <td>will be moving</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>i paid the bill it reflected on my bank statem...</td>\n",
              "      <td>pay bill</td>\n",
              "      <td>it did not</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>i 'm filing my taxes today and i need your hel...</td>\n",
              "      <td>help with my taxes</td>\n",
              "      <td>need help with my phone plan payments</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>good afternoon freedom mobile customer service...</td>\n",
              "      <td>block number</td>\n",
              "      <td>can you please block my number</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>please block my phone number from scammer</td>\n",
              "      <td>block number</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>i would like to pay my phone bill</td>\n",
              "      <td>pay bill</td>\n",
              "      <td>no context given</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>how much money do i need to pay for my top up</td>\n",
              "      <td>top up</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>9</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>i want to close my account</td>\n",
              "      <td>close account</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>i just got my sim but your website is n't working</td>\n",
              "      <td>sim issue</td>\n",
              "      <td>not working</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>my phone line is not working</td>\n",
              "      <td>phone line issue</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>can't make calls</td>\n",
              "      <td>can't make calls</td>\n",
              "      <td>can't make calls</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>i did not receive an email or text to notify me</td>\n",
              "      <td>text</td>\n",
              "      <td>not notified</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>9</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>i have always been contated but not ts month</td>\n",
              "      <td>contatc</td>\n",
              "      <td>not</td>\n",
              "      <td>-1</td>\n",
              "      <td>6</td>\n",
              "      <td>9</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>why did i not received a notice</td>\n",
              "      <td>notify</td>\n",
              "      <td>not given</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>i would like to change the address for my home...</td>\n",
              "      <td>change address</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>what is the process to change my plan into pos...</td>\n",
              "      <td>change plan</td>\n",
              "      <td>what is the process to change my plan into pos...</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 text  ... grouped_topics_hdbscan_reduced\n",
              "0                                   cancel my account  ...                              2\n",
              "1                              i want to add new line  ...                              1\n",
              "2                                 speak with an agent  ...                              2\n",
              "3   am i able to cancel my plan and close ts numbe...  ...                              2\n",
              "4   unfortunately i am moving somewhere without fr...  ...                              0\n",
              "5   i paid the bill it reflected on my bank statem...  ...                             -1\n",
              "6   i 'm filing my taxes today and i need your hel...  ...                              0\n",
              "7   good afternoon freedom mobile customer service...  ...                              1\n",
              "8           please block my phone number from scammer  ...                              1\n",
              "9                   i would like to pay my phone bill  ...                             -1\n",
              "10      how much money do i need to pay for my top up  ...                              2\n",
              "11                         i want to close my account  ...                              2\n",
              "12  i just got my sim but your website is n't working  ...                              2\n",
              "13                       my phone line is not working  ...                              1\n",
              "14                                   can't make calls  ...                              2\n",
              "15    i did not receive an email or text to notify me  ...                              0\n",
              "16       i have always been contated but not ts month  ...                              2\n",
              "17                    why did i not received a notice  ...                              1\n",
              "18  i would like to change the address for my home...  ...                              1\n",
              "19  what is the process to change my plan into pos...  ...                             -1\n",
              "\n",
              "[20 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bjB_pOMWwTL8"
      },
      "source": [
        "## clustering - KMeans"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NE5LDqi0wOAq"
      },
      "source": [
        "from sklearn.cluster import KMeans\n",
        "\n",
        "kmeans = KMeans(n_clusters=10).fit(embeddings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zIordAIVwsim",
        "outputId": "df118f65-0e57-489a-e61d-d6a8e0e41431"
      },
      "source": [
        "kmeans.labels_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([6, 4, 9, 1, 3, 7, 3, 4, 4, 7, 0, 6, 8, 4, 1, 3, 4, 4, 5, 2, 3, 3,\n",
              "       9, 2, 7, 1, 8, 5, 4, 3, 3, 9, 8, 8, 4, 5, 2, 7, 7, 8, 1, 8, 6, 7,\n",
              "       3, 4, 7, 9, 7, 7, 5, 5, 9, 1, 6, 4, 3, 3, 7, 1, 8, 3, 3, 4, 0, 2,\n",
              "       4, 4, 2, 2, 8, 9, 6, 5, 1, 2, 4, 7, 5, 1, 1, 7, 3, 4, 2, 2, 3, 1,\n",
              "       8, 8, 3, 5, 5, 5, 6, 7, 2, 4, 3, 4, 5, 5, 1, 2, 0, 7, 1, 4, 3, 6,\n",
              "       0, 6, 6, 1, 6, 6, 4, 2], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AZnv1eHNwkPa"
      },
      "source": [
        "kmeans_reduced = KMeans(n_clusters=10).fit(embeddings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RmF3g4b2wqFi",
        "outputId": "557abb6b-7a8e-4c19-e528-a17d4169e2f5"
      },
      "source": [
        "kmeans_reduced.labels_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([3, 8, 6, 3, 2, 4, 2, 5, 5, 4, 8, 3, 9, 5, 8, 8, 8, 8, 7, 1, 2, 2,\n",
              "       6, 1, 4, 3, 9, 7, 8, 2, 2, 6, 9, 9, 0, 7, 1, 4, 4, 9, 8, 9, 8, 0,\n",
              "       2, 8, 4, 6, 4, 4, 7, 7, 6, 3, 3, 5, 2, 2, 4, 6, 9, 2, 2, 5, 8, 1,\n",
              "       8, 8, 1, 1, 9, 6, 8, 7, 3, 1, 8, 4, 7, 3, 3, 4, 2, 7, 1, 1, 2, 6,\n",
              "       9, 9, 2, 7, 7, 7, 3, 4, 1, 5, 2, 0, 7, 7, 3, 1, 8, 4, 3, 7, 2, 3,\n",
              "       8, 3, 3, 3, 3, 3, 5, 1], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MPvA8NEfwuzv"
      },
      "source": [
        "df[\"grouped_topics_kmeans\"] = kmeans.labels_\n",
        "df[\"grouped_topics_kmeans_reduced\"] = kmeans_reduced.labels_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "rgDvMrbfxwlt",
        "outputId": "4f14109d-9a96-40e3-a3c7-6c20d5a897f9"
      },
      "source": [
        "df.head(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>class</th>\n",
              "      <th>context</th>\n",
              "      <th>grouped_topics</th>\n",
              "      <th>grouped_topics_kmeans</th>\n",
              "      <th>grouped_topics_kmeans_reduced</th>\n",
              "      <th>grouped_topics_hdbscan</th>\n",
              "      <th>grouped_topics_hdbscan_reduced</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cancel my account</td>\n",
              "      <td>cancel account</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i want to add new line</td>\n",
              "      <td>add new line</td>\n",
              "      <td>no context given</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>speak with an agent</td>\n",
              "      <td>speak agent</td>\n",
              "      <td>no context given</td>\n",
              "      <td>-1</td>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>am i able to cancel my plan and close ts numbe...</td>\n",
              "      <td>cancel plan</td>\n",
              "      <td>not here</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>unfortunately i am moving somewhere without fr...</td>\n",
              "      <td>help</td>\n",
              "      <td>will be moving</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>i paid the bill it reflected on my bank statem...</td>\n",
              "      <td>pay bill</td>\n",
              "      <td>it did not</td>\n",
              "      <td>-1</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>i 'm filing my taxes today and i need your hel...</td>\n",
              "      <td>help with my taxes</td>\n",
              "      <td>need help with my phone plan payments</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>good afternoon freedom mobile customer service...</td>\n",
              "      <td>block number</td>\n",
              "      <td>can you please block my number</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>please block my phone number from scammer</td>\n",
              "      <td>block number</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>i would like to pay my phone bill</td>\n",
              "      <td>pay bill</td>\n",
              "      <td>no context given</td>\n",
              "      <td>-1</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>how much money do i need to pay for my top up</td>\n",
              "      <td>top up</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>i want to close my account</td>\n",
              "      <td>close account</td>\n",
              "      <td>no context given</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>i just got my sim but your website is n't working</td>\n",
              "      <td>sim issue</td>\n",
              "      <td>not working</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>9</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>my phone line is not working</td>\n",
              "      <td>phone line issue</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>can't make calls</td>\n",
              "      <td>can't make calls</td>\n",
              "      <td>can't make calls</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>i did not receive an email or text to notify me</td>\n",
              "      <td>text</td>\n",
              "      <td>not notified</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>i have always been contated but not ts month</td>\n",
              "      <td>contatc</td>\n",
              "      <td>not</td>\n",
              "      <td>-1</td>\n",
              "      <td>4</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>why did i not received a notice</td>\n",
              "      <td>notify</td>\n",
              "      <td>not given</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>8</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>i would like to change the address for my home...</td>\n",
              "      <td>change address</td>\n",
              "      <td>no context given</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>7</td>\n",
              "      <td>-1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>what is the process to change my plan into pos...</td>\n",
              "      <td>change plan</td>\n",
              "      <td>what is the process to change my plan into pos...</td>\n",
              "      <td>-1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 text  ... grouped_topics_hdbscan_reduced\n",
              "0                                   cancel my account  ...                              2\n",
              "1                              i want to add new line  ...                              1\n",
              "2                                 speak with an agent  ...                              2\n",
              "3   am i able to cancel my plan and close ts numbe...  ...                              2\n",
              "4   unfortunately i am moving somewhere without fr...  ...                              0\n",
              "5   i paid the bill it reflected on my bank statem...  ...                             -1\n",
              "6   i 'm filing my taxes today and i need your hel...  ...                              0\n",
              "7   good afternoon freedom mobile customer service...  ...                              1\n",
              "8           please block my phone number from scammer  ...                              1\n",
              "9                   i would like to pay my phone bill  ...                             -1\n",
              "10      how much money do i need to pay for my top up  ...                              2\n",
              "11                         i want to close my account  ...                              2\n",
              "12  i just got my sim but your website is n't working  ...                              2\n",
              "13                       my phone line is not working  ...                              1\n",
              "14                                   can't make calls  ...                              2\n",
              "15    i did not receive an email or text to notify me  ...                              0\n",
              "16       i have always been contated but not ts month  ...                              2\n",
              "17                    why did i not received a notice  ...                              1\n",
              "18  i would like to change the address for my home...  ...                              1\n",
              "19  what is the process to change my plan into pos...  ...                             -1\n",
              "\n",
              "[20 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pJ-6zbBqx2yB"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}