NLP_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_Project.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "DOzvReJYLhY8",
        "m1PbZaGLq_2_",
        "RIkUWzYnL7bm",
        "rGMqqO97rW7c",
        "5ls7MUY_Dgub",
        "FRnj56UiXnk0",
        "BtqLm1bWX7hS",
        "rQda7_PwRN_L",
        "0fR3yjzNwYrU",
        "lubjtTj1LtCX"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "cb13f7f40a024af5873336daa3f3c6f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_359482ceab754516a85d8f82a3ddc2f3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_370fa33a40fb4bf4919432b1e92cb29a",
              "IPY_MODEL_9947f87aebc640c8ba86ea466333118a"
            ]
          }
        },
        "359482ceab754516a85d8f82a3ddc2f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "370fa33a40fb4bf4919432b1e92cb29a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88ee5227f99c480d85a39af8ed83bdd8",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 414,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 414,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f999763278b471597eea56c2a2db0bc"
          }
        },
        "9947f87aebc640c8ba86ea466333118a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fcc38b513ce94551a02f45288ca51330",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 414/414 [00:00&lt;00:00, 1.85kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b9c1f209af644c4bbd838a08b3fae6cd"
          }
        },
        "88ee5227f99c480d85a39af8ed83bdd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f999763278b471597eea56c2a2db0bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fcc38b513ce94551a02f45288ca51330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b9c1f209af644c4bbd838a08b3fae6cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd7399da2c944392bf27bdd98de14da2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_396f585ad20044d1acc84d1558649585",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ee41aa48bf4d4ab5a5bba1ae9649a71c",
              "IPY_MODEL_1df27c9a9d4243dd96232ba250df5841"
            ]
          }
        },
        "396f585ad20044d1acc84d1558649585": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee41aa48bf4d4ab5a5bba1ae9649a71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b224e0d4659a45c1b38a4f71540c6432",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 665132540,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 665132540,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b1cd11c562b04dabb98e44d81f9efeb6"
          }
        },
        "1df27c9a9d4243dd96232ba250df5841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7a5caeda0c5a48aead8771116ad5528d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 665M/665M [00:15&lt;00:00, 42.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cee6f845c0874f0f82cb85f0f05564fb"
          }
        },
        "b224e0d4659a45c1b38a4f71540c6432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b1cd11c562b04dabb98e44d81f9efeb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a5caeda0c5a48aead8771116ad5528d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cee6f845c0874f0f82cb85f0f05564fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c74c07c2d2ef40969d0417c1b4140be9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f514a66213454115b86ae8867bbba6b2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d6276bfbb537466b827e0cd4ada9ce1e",
              "IPY_MODEL_85f0cf66d7da456fb4c3f8ada8c0c1b7"
            ]
          }
        },
        "f514a66213454115b86ae8867bbba6b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6276bfbb537466b827e0cd4ada9ce1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_247c518e543e4853854dff4ebdfaf44d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcdfd98b4bdb4ae8839097cb2ddf8979"
          }
        },
        "85f0cf66d7da456fb4c3f8ada8c0c1b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95abdb6b947944c4892c7f317e1b1390",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 619kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3535a5a3f9414362b290ec7b6c20347f"
          }
        },
        "247c518e543e4853854dff4ebdfaf44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcdfd98b4bdb4ae8839097cb2ddf8979": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95abdb6b947944c4892c7f317e1b1390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3535a5a3f9414362b290ec7b6c20347f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8326689e5a334f8a8555ce223115090d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_656d2cbcd6704b4bb32efb6c7c6d8b27",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ddd553093fd6475ba5182d7f43ab8534",
              "IPY_MODEL_c791108e80f84afab8fcd3dac0d1c119"
            ]
          }
        },
        "656d2cbcd6704b4bb32efb6c7c6d8b27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ddd553093fd6475ba5182d7f43ab8534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b9b554f7b2a94943bb5dcca7de8a4b7c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e804fb73b9d4238bbc524973e1736ec"
          }
        },
        "c791108e80f84afab8fcd3dac0d1c119": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4d0b323f82db4b34844f23d010a11eea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:07&lt;00:00, 60.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_221ad883825d47ceae400b53407a9c6c"
          }
        },
        "b9b554f7b2a94943bb5dcca7de8a4b7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e804fb73b9d4238bbc524973e1736ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d0b323f82db4b34844f23d010a11eea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "221ad883825d47ceae400b53407a9c6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fea0e9192f304cd8afa068c71c206133": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bf4cbe43e8bd432fadbd458ca102e292",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d35361602ac14debb6ef277324568d31",
              "IPY_MODEL_3261ee26d59b442e9a70a31511af82dc"
            ]
          }
        },
        "bf4cbe43e8bd432fadbd458ca102e292": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d35361602ac14debb6ef277324568d31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2505316b5c914dc98719ead80735a0f7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_002675c246574e7384172d701f4737a4"
          }
        },
        "3261ee26d59b442e9a70a31511af82dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ca0a81d23a4b4be6b3c4830516c4d74a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 63.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cd860c5a3110457fa4107fdd255a537a"
          }
        },
        "2505316b5c914dc98719ead80735a0f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "002675c246574e7384172d701f4737a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca0a81d23a4b4be6b3c4830516c4d74a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cd860c5a3110457fa4107fdd255a537a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da7ad6d8b68a48e28266068afdcf38ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_295394ec4afc4b519cce7ab5c4148841",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d41474395d7a4d3db1dc0036816e3409",
              "IPY_MODEL_e6da87c9dbc84c3bb5efdaad63c6d3c8"
            ]
          }
        },
        "295394ec4afc4b519cce7ab5c4148841": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d41474395d7a4d3db1dc0036816e3409": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8aa0355ba33542a98b4abf3d31694ead",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e074239208c5463a8324a2600cea5e7c"
          }
        },
        "e6da87c9dbc84c3bb5efdaad63c6d3c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fb36f3220ed54e40b7a40f89fe257c9c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 2.84MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cf0b7c632b224b299b4215cb10ae5854"
          }
        },
        "8aa0355ba33542a98b4abf3d31694ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e074239208c5463a8324a2600cea5e7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb36f3220ed54e40b7a40f89fe257c9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cf0b7c632b224b299b4215cb10ae5854": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DOzvReJYLhY8",
        "colab_type": "text"
      },
      "source": [
        "# Setup\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "snKYGjfZTWGI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -U spacy\n",
        "import nltk\n",
        "import spacy\n",
        "from spacy.lang.en import English\n",
        "!python -m spacy download en_core_web_md\n",
        "!python -m spacy download en_core_web_sm-2.2.0 --direct\n",
        "!pip install allennlp==1.0.0rc1 allennlp-models==1.0.0rc1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsyeH8e3waYy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import word_tokenize\n",
        "import pdb\n",
        "import itertools\n",
        "from itertools import permutations\n",
        "from tqdm import tqdm\n",
        "from nltk.corpus import stopwords\n",
        "import string \n",
        "from difflib import SequenceMatcher\n",
        "from spacy.pipeline import EntityRuler\n",
        "nltk.download('popular')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Obh3w3BL514q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.corpus import wordnet as wn\n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
        "import re\n",
        "from queue import Queue \n",
        "from spacy.matcher import PhraseMatcher\n",
        "import json\n",
        "import csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m1PbZaGLq_2_",
        "colab_type": "text"
      },
      "source": [
        "# READ FILE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mKvokS01VgkP",
        "colab_type": "code",
        "outputId": "6fd0ea87-aa3b-42cf-e73e-71d20397c769",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-63b727c2-a323-4cb4-9a9d-0cb503e7aca0\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-63b727c2-a323-4cb4-9a9d-0cb503e7aca0\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving LaurenePowellJobs.txt to LaurenePowellJobs.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7BtaycYrfam",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for name, data in uploaded.items():\n",
        "  filename = name\n",
        "\n",
        "input_f = open(filename, 'r', encoding='utf-8')\n",
        "content = input_f.read()\n",
        "input_f.close()\n",
        "output_filename = \"output\" + filename.split(\".txt\")[0] + \".json\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RIkUWzYnL7bm",
        "colab_type": "text"
      },
      "source": [
        "# Set Up all JOB Titles"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFjtYlE1MC8M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Fetching Job Titles from file and making a dictionary out of it\n",
        "job_title_content=open(\"job_title_dataset.csv\",\"r+\")\n",
        "job_title_data=csv.reader(job_title_content)\n",
        "job_title=set()\n",
        "flag=True;\n",
        "for row in job_title_data:\n",
        "  if flag:\n",
        "    flag=False\n",
        "    continue\n",
        "  job_title.add(row[0].lower())\n",
        "job_title_content.close()\n",
        "job_title.discard(\"\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7KvQMw0Miqi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def patternList(job_title):\n",
        "  '''Storing Job titles dictionary in form of patterns List which is used for Entity Ruler of Spacy\n",
        "    \n",
        "    Args:\n",
        "      job_title (dict): Dictionary of all the Job Titles\n",
        "\n",
        "    Returns:\n",
        "      final_pattern (list): List of all the Job Titles patterns generated \n",
        "  '''\n",
        "  job_title.update([\"board of directors\", \"chairman's advisory board\", \"buisness trading strategist\" ])\n",
        "  final_pattern = []\n",
        "  for title in job_title:\n",
        "    #removing garbage titles\n",
        "    if len(title) <= 2:\n",
        "      continue\n",
        "    pattern = {\"label\": \"TITLE\", \"pattern\": \"\"}\n",
        "\n",
        "    tokens = tokenizer(title)\n",
        "    lower_list = []\n",
        "    for title_sub in tokens:\n",
        "      lower = {\"LOWER\": title_sub.text.lower()}\n",
        "      lower_list.append(lower)\n",
        "    pattern[\"pattern\"] = lower_list\n",
        "\n",
        "    final_pattern.append(pattern) \n",
        "  return final_pattern"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rGMqqO97rW7c",
        "colab_type": "text"
      },
      "source": [
        "# Part(Location, Location) Template"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OxGr79RmtVDF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getNERFeatures(doc):\n",
        "  '''Retrieve all the Named Entity Recognition Features from sentence doc (spacy doc) and storing into dictionary\n",
        "    \n",
        "    Args:\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline \n",
        "\n",
        "    Returns:\n",
        "      nerDict (dict): Named Entity Recognition Dictionary \n",
        "  ''' \n",
        "\n",
        "  nerDict = {}\n",
        "  for x in doc.ents:\n",
        "    nerDict[x.text]=x.label_\n",
        "  return nerDict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-8UCYaQteWM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getWordnetFeatures(doc):\n",
        "  '''Retrieve WordNet feature from sentence doc (spacy doc) and storing into dictionary\n",
        "    \n",
        "    Args:\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline \n",
        "\n",
        "    Returns:\n",
        "      wordnetTokenDict (dict): WordNet Feature Dictionary\n",
        "  ''' \n",
        "  wordnetTokenDict = {}\n",
        "  tokens = []\n",
        "\n",
        "  for ent in doc.ents:\n",
        "    text = ent.text\n",
        "    #convert two words sperate by '_' in order to get correct info from nltk wordnet\n",
        "    #eg. San Francisco -> San_Francisco \n",
        "    text = text.replace(' ', '_')\n",
        "    tokens.append(text)\n",
        "\n",
        "  for token in tokens:\n",
        "    tokenSynset = wn.synsets(token)\n",
        "    synonyms=[]\n",
        "    hypernymList=[]\n",
        "    hyponymList=[]\n",
        "    holonymList=[]\n",
        "    meronymList=[]\n",
        "    entailList=[]\n",
        "    memberHolonymsList = []\n",
        "  \n",
        "    tempTokenDict={}\n",
        "    for index, tempSynset in enumerate(tokenSynset):\n",
        "      \n",
        "      hypernyms = tempSynset.hypernyms()\n",
        "      for l in hypernyms:\n",
        "        hypernymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      hyponyms = tempSynset.hyponyms()\n",
        "      for l in hyponyms:\n",
        "        hyponymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      holonyms = tempSynset.part_holonyms()\n",
        "      for l in holonyms:\n",
        "        holonymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      meronyms = tempSynset.part_meronyms()\n",
        "      for l in meronyms:\n",
        "        meronymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      entail = tempSynset.entailments()\n",
        "      for l in entail:\n",
        "        entailList.append(l.lemma_names()[0])\n",
        "        \n",
        "      for l in tempSynset.lemmas():\n",
        "        synonyms.append(l.name())\n",
        "      \n",
        "      memberHolonyms = tempSynset.member_holonyms()\n",
        "      for l in memberHolonyms:\n",
        "        memberHolonymsList.append(l.name())\n",
        "\n",
        "      \n",
        "    tempTokenDict[\"hypernym\"]=hypernymList  \n",
        "    tempTokenDict[\"hyponym\"]=hyponymList\n",
        "    tempTokenDict[\"holonym\"]=holonymList\n",
        "    tempTokenDict[\"meronym\"]=meronymList\n",
        "    tempTokenDict[\"entail\"]=entailList\n",
        "    tempTokenDict[\"synonym\"]=synonyms\n",
        "    tempTokenDict[\"member_holonyms\"] = memberHolonymsList\n",
        "\n",
        "    \n",
        "    wordnetTokenDict[token.replace('_', ' ')]=tempTokenDict    \n",
        "  \n",
        "  return wordnetTokenDict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqbi4f_Uq9F7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def partTemplate(sent, link_coref_sent, doc, EXTRACTION_LIST):\n",
        "  '''Extracting Part(Location, Location) template from the sentence and storing\n",
        "    all the extracted templates into EXTRACTION_LIST\n",
        "    \n",
        "    Args:\n",
        "      sent (string): Sentence on which extraction is performed\n",
        "      link_coref_sent (string): Orignal set of sentences from where correference\n",
        "                                of current sentence is resolved.\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "      EXTRACTION_LIST (list): Previously extracted templates list to which new templates will be appended\n",
        "  ''' \n",
        "  #Fetching NER and WordNet Features\n",
        "  nerDict = getNERFeatures(doc)\n",
        "  wordnetFeatures = getWordnetFeatures(doc)\n",
        "\n",
        "  part_of_template = set()\n",
        "  possibleLocations = []\n",
        "  for i in nerDict:\n",
        "    if nerDict[i] == 'GPE' or nerDict[i] == 'LOC' or nerDict[i] == 'FAC' or nerDict[i] == 'NORP':\n",
        "      possibleLocations.append(i)\n",
        "\n",
        "  # If possible locations is less than 2 then we cannot find relation so return      \n",
        "  if len(possibleLocations) < 2:\n",
        "    return\n",
        "  \n",
        "  perm = list(permutations(possibleLocations, 2))\n",
        "  \n",
        "  #check direct relation by checking left's synonyms -> meronym of right\n",
        "  for tuple in perm:\n",
        "    left = tuple[0]\n",
        "    right = tuple[1]\n",
        "\n",
        "    leftSynonymSet = set([left.replace('', '_')])\n",
        "    rightMeronymSet = set()\n",
        "\n",
        "    leftSynonymSet.update(wordnetFeatures[left][\"synonym\"])\n",
        "    rightMeronymSet.update(wordnetFeatures[right][\"meronym\"])\n",
        "\n",
        "    intersectionSet = leftSynonymSet.intersection(rightMeronymSet)\n",
        "    if len(intersectionSet) > 0:\n",
        "      part_of_template.add(tuple)\n",
        "\n",
        "  #check direct relation by checking left's holonym -> synonym of right\n",
        "  for tuple in perm:\n",
        "    left = tuple[0]\n",
        "    right = tuple[1]\n",
        "\n",
        "    leftHolonymSet = set()\n",
        "    rightSynonymSet = set([right.replace(' ', '_')])\n",
        "\n",
        "    leftHolonymSet.update(wordnetFeatures[left][\"holonym\"])\n",
        "    rightSynonymSet.update(wordnetFeatures[right][\"synonym\"])\n",
        "\n",
        "    intersectionSet = leftHolonymSet.intersection(rightSynonymSet)\n",
        "    if len(intersectionSet) > 0:\n",
        "      part_of_template.add(tuple)\n",
        "  \n",
        "  #check for transitive relations\n",
        "  #i.e., left's holonym -> right's meronym\n",
        "  for tuple in perm:\n",
        "    left = tuple[0]\n",
        "    right = tuple[1]\n",
        "\n",
        "    leftHolonymSet = set()\n",
        "    rightMeronymSet = set()\n",
        "\n",
        "    leftHolonymSet.update(wordnetFeatures[left][\"holonym\"])\n",
        "    rightMeronymSet.update(wordnetFeatures[right][\"meronym\"])\n",
        "\n",
        "    intersectionSet = leftHolonymSet.intersection(rightMeronymSet)\n",
        "    if len(intersectionSet) > 0:\n",
        "      part_of_template.add(tuple)\n",
        "\n",
        "  location_templates = []\n",
        "  for tup in part_of_template:\n",
        "    location_templates.append({\"Location1\": tup[0], \"Location2\": tup[1]})\n",
        "  \n",
        "  for temp in location_templates:\n",
        "    final_part_template_format = {\"template\": \"PART\", \n",
        "                                \"sentences\": [link_coref_sent], \n",
        "                                \"arguements\": {\"1\": \"\",\n",
        "                                          \"2\": \"\"\n",
        "                                          }\n",
        "                          } \n",
        "    final_part_template_format[\"arguements\"][\"1\"] = temp[\"Location1\"]\n",
        "    final_part_template_format[\"arguements\"][\"2\"] = temp[\"Location2\"]\n",
        "    EXTRACTION_LIST.append(final_part_template_format)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ls7MUY_Dgub",
        "colab_type": "text"
      },
      "source": [
        "# BUY Template"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mEXrTfT2EqI-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getLemma(verb):\n",
        "  '''getting Lemma form of the input verb\n",
        "    \n",
        "    Args:\n",
        "      verb (string): verb \n",
        "\n",
        "    Returns:\n",
        "      lemmas (set): Lemmas of input verb \n",
        "  '''\n",
        "  doc = nlp_other(verb)\n",
        "\n",
        "  lemmas = set()\n",
        "  for token in doc:\n",
        "    lemmas.add(token.lemma_)\n",
        "  \n",
        "  return lemmas"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8spsYW_NHax",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getSRLArguements(srl_obj):\n",
        "  ''' Extract SRL Arguements and its value from SRL Object\n",
        "      example the srl_obj[\"description\"] = [ARG0: Amazon] [V: bought] [ARG1: Whole Food Market] [ARG3: for $ 1 Million Dollars] .\n",
        "      so, we will extract the arguements and its value and store into arguement_map.\n",
        "      Output: {\"ARG0\": \"Amazon\", \"ARG1\": \"Whole Food Market\", \"ARG3\": \"for $ 1 Million Dollars\"}\n",
        "    \n",
        "    Args:\n",
        "      srl_obj (object): Semanctic Role Labelling Object obtained by applying \n",
        "                        AllenNlp SRL \n",
        "\n",
        "    Returns:\n",
        "      arguement_map (dict): SRL Arguements extracted from srl_obj\n",
        "  '''\n",
        "\n",
        "  description = srl_obj[\"description\"]\n",
        "  arguements_box = re.findall(\"\\[[^\\]]*\\]\", description)\n",
        "\n",
        "  arguement_map = {}\n",
        "\n",
        "  for box in arguements_box:\n",
        "    if len(box.split(\":\")) > 1:\n",
        "      arguement, text = box.split(\":\")[0][1:], box.split(\":\")[1][:-1]\n",
        "\n",
        "      if len(arguement) == 0 or len(text) == 0:\n",
        "        continue\n",
        "    arguement_map[arguement] = text.strip()\n",
        "  \n",
        "  return arguement_map\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7sKikpXZDzy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def buyTemplate(new_sent, link_coref_sent, srl_tags, doc1, EXTRACTION_LIST):\n",
        "  ''' Extracting Buy(Buyer, Item, Price, Quantity, Source) template from the sentence and storing\n",
        "    all the extracted templates into EXTRACTION_LIST\n",
        "    \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      link_coref_sent (string): Orignal set of sentences from where correference\n",
        "                                of current sentence is resolved.\n",
        "      srl_tags (object): SRL Tags obtained by performing AllenNlp SRL model on sentence\n",
        "      doc1 (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "      EXTRACTION_LIST (list): Previously extracted templates list to which new templates will be appended\n",
        "  '''\n",
        "  #set of buy and sell verbs which helps to trigger buying and selling event\n",
        "  buy_verbs_set = set([\"buy\", \"purchase\", \"get\", \"acquire\", \"capture\", \"take\", \"obtain\", \"retrive\"])\n",
        "  sell_verbs_set = set([\"sell\"])\n",
        "\n",
        "  srl_verb_obj_list = srl_tags[\"verbs\"]\n",
        "\n",
        "  final_template_format = {\"template\": \"BUY\", \n",
        "                           \"sentences\": [link_coref_sent], \n",
        "                           \"arguements\": {\"1\": \"\",\n",
        "                                          \"2\": \"\",\n",
        "                                          \"3\": \"\",\n",
        "                                          \"4\": \"\",\n",
        "                                          \"5\": \"\"\n",
        "                                          }\n",
        "                          }   \n",
        "  #spacy en_core_web_sm model\n",
        "  doc2 = nlp_other(new_sent)\n",
        "\n",
        "  buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "  for srl_verb_obj in srl_verb_obj_list:\n",
        "    #get lemma form of verb to check with our buy_verbs_set and sell_verbs_set\n",
        "    verb_lemmas = getLemma(srl_verb_obj[\"verb\"])\n",
        "\n",
        "    #Buy verb\n",
        "    if len(buy_verbs_set.intersection(verb_lemmas)) > 0:\n",
        "\n",
        "      arguement_map = getSRLArguements(srl_verb_obj)\n",
        "\n",
        "      #running on sm and md model to fetch more results\n",
        "      template1 = buy_md_test(new_sent, arguement_map, buy_template, doc1)\n",
        "      template2 = buy_sm_test(new_sent, arguement_map, buy_template, doc2)\n",
        "      buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "\n",
        "      for key in buy_template:\n",
        "        if len(template1[key]) >= len(template2[key]):\n",
        "          buy_template[key] = template1[key]\n",
        "        else:\n",
        "          buy_template[key] = template2[key]\n",
        "\n",
        "      if buy_template[\"buyer\"] != \"\" and buy_template[\"item\"] != \"\":\n",
        "        final_template_format[\"arguements\"][\"1\"] = buy_template[\"buyer\"]\n",
        "        final_template_format[\"arguements\"][\"2\"] = buy_template[\"item\"]\n",
        "        final_template_format[\"arguements\"][\"3\"] = buy_template[\"price\"]\n",
        "        final_template_format[\"arguements\"][\"4\"] = buy_template[\"quantity\"]\n",
        "        final_template_format[\"arguements\"][\"5\"] = buy_template[\"source\"]\n",
        "\n",
        "        EXTRACTION_LIST.append(final_template_format)      \n",
        "\n",
        "    \n",
        "    #Sell verb\n",
        "    if len(sell_verbs_set.intersection(verb_lemmas)) > 0:\n",
        "\n",
        "      arguement_map = getSRLArguements(srl_verb_obj)\n",
        "\n",
        "      #running on sm and md model to fetch more results\n",
        "      template1 = sell_md_test(new_sent, arguement_map, buy_template, doc1)\n",
        "      template2 = sell_sm_test(new_sent, arguement_map, buy_template, doc2)\n",
        "\n",
        "      buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "\n",
        "      for key in buy_template:\n",
        "        if len(template1[key]) >= len(template2[key]):\n",
        "          buy_template[key] = template1[key]\n",
        "        else:\n",
        "          buy_template[key] = template2[key]\n",
        "\n",
        "      if buy_template[\"buyer\"] != \"\" and buy_template[\"item\"] != \"\":\n",
        "        final_template_format[\"arguements\"][\"1\"] = buy_template[\"buyer\"]\n",
        "        final_template_format[\"arguements\"][\"2\"] = buy_template[\"item\"]\n",
        "        final_template_format[\"arguements\"][\"3\"] = buy_template[\"price\"]\n",
        "        final_template_format[\"arguements\"][\"4\"] = buy_template[\"quantity\"]\n",
        "        final_template_format[\"arguements\"][\"5\"] = buy_template[\"source\"]\n",
        "\n",
        "        EXTRACTION_LIST.append(final_template_format)      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CsloYCURbAV4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def buy_sm_test(new_sent, arguement_map, buy_template, doc):\n",
        "  ''' Extracting Buy(Buyer, Item, Price, Quantity, Source) template from the sentence \n",
        "      by running spacy's sm model.\n",
        "      This is called when Buying event is triggered.\n",
        "      \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      arguement_map (dict): SRL obj arguement dictionary\n",
        "      buy_template (dict): buy template blank format which needs to be filled\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "    \n",
        "    Returns:\n",
        "      buy_template (dict): BUY template extracted\n",
        "  '''\n",
        "  buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "  \n",
        "  if \"ARG0\" in arguement_map:\n",
        "        \n",
        "    #extracting BUYER from ARG0 and SELLER from ARG2\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and ent.text in arguement_map[\"ARG0\"]:\n",
        "        buy_template[\"buyer\"] = ent.text\n",
        "      elif ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and \"ARG2\" in arguement_map and ent.text in arguement_map[\"ARG2\"]:\n",
        "        buy_template[\"source\"] = ent.text\n",
        "    \n",
        "    \n",
        "    #extracting ITEM bought from ARG1\n",
        "    item_list=[]\n",
        "    for ent in doc.ents:\n",
        "      # print(ent)\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"ORG\", \"PRODUCT\"] and \"ARG1\" in arguement_map and ent.text in arguement_map[\"ARG1\"]:\n",
        "        item_list.append(ent.text)\n",
        "    buy_template[\"item\"] = \";\".join(item_list)\n",
        "    \n",
        "    #extracting MONEY from ARG3 if available\n",
        "    for ent in doc.ents:\n",
        "      \n",
        "      ent_tokenize = set(word_tokenize(ent.text))\n",
        "\n",
        "      if ent.label_ == \"MONEY\":\n",
        "        if \"ARG3\" in arguement_map:\n",
        "          arg3_tokenize = set(word_tokenize(arguement_map[\"ARG3\"]))\n",
        "          if len(ent_tokenize.intersection(arg3_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "        else:\n",
        "          arg1_tokenize = set(word_tokenize(arguement_map[\"ARG1\"]))\n",
        "          if len(ent_tokenize.intersection(arg1_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "    \n",
        "    #extracting QUANTITY if present by checking NER of Quantity or Cardinal\n",
        "    for ent in doc.ents:\n",
        "      quantity_ner = []\n",
        "      cardinal_ner = []\n",
        "      for ent in doc.ents:\n",
        "        if ent.label_ == \"QUANTITY\":\n",
        "          quantity_ner.append(ent.text)\n",
        "        elif ent.label_ == \"CARDINAL\":\n",
        "          cardinal_ner.append(ent.text)\n",
        "      \n",
        "      if len(quantity_ner) > 0:\n",
        "        buy_template[\"quantity\"] = \";\".join(quantity_ner)\n",
        "      else:\n",
        "        buy_template[\"quantity\"] = \";\".join(cardinal_ner)\n",
        "  return buy_template\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FEZbM2THatFg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def buy_md_test(new_sent, arguement_map, buy_template, doc):\n",
        "  ''' Extracting Buy(Buyer, Item, Price, Quantity, Source) template from the sentence \n",
        "      by running spacy's md model.\n",
        "      This is called when Buying event is triggered.\n",
        "      \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      arguement_map (dict): SRL obj arguement dictionary\n",
        "      buy_template (dict): buy template blank format which needs to be filled\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "    \n",
        "    Returns:\n",
        "      buy_template (dict): BUY template extracted\n",
        "  '''\n",
        "  buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "  \n",
        "  if \"ARG0\" in arguement_map:\n",
        "        \n",
        "    #extracting BUYER from ARG0 and SELLER from ARG2\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and ent.text in arguement_map[\"ARG0\"]:\n",
        "        buy_template[\"buyer\"] = ent.text\n",
        "      elif ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and \"ARG2\" in arguement_map and ent.text in arguement_map[\"ARG2\"]:\n",
        "        buy_template[\"source\"] = ent.text\n",
        "    \n",
        "    \n",
        "    #extracting ITEM bought from ARG1\n",
        "    item_list=[]\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"ORG\", \"PRODUCT\"] and \"ARG1\" in arguement_map and ent.text in arguement_map[\"ARG1\"]:\n",
        "        item_list.append(ent.text)\n",
        "    buy_template[\"item\"] = \";\".join(item_list)\n",
        "    \n",
        "    #extracting MONEY from ARG3 if available\n",
        "    for ent in doc.ents:      \n",
        "      ent_tokenize = set(word_tokenize(ent.text))\n",
        "\n",
        "      if ent.label_ == \"MONEY\":\n",
        "        if \"ARG3\" in arguement_map:\n",
        "          arg3_tokenize = set(word_tokenize(arguement_map[\"ARG3\"]))\n",
        "          if len(ent_tokenize.intersection(arg3_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "        else:\n",
        "          arg1_tokenize = set(word_tokenize(arguement_map[\"ARG1\"]))\n",
        "          if len(ent_tokenize.intersection(arg1_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "    \n",
        "    #extracting QUANTITY if present by checking NER of Quantity or Cardinal\n",
        "    for ent in doc.ents:\n",
        "      quantity_ner = []\n",
        "      cardinal_ner = []\n",
        "      for ent in doc.ents:\n",
        "        if ent.label_ == \"QUANTITY\":\n",
        "          quantity_ner.append(ent.text)\n",
        "        elif ent.label_ == \"CARDINAL\":\n",
        "          cardinal_ner.append(ent.text)\n",
        "      \n",
        "      if len(quantity_ner) > 0:\n",
        "        buy_template[\"quantity\"] = \";\".join(quantity_ner)\n",
        "      else:\n",
        "        buy_template[\"quantity\"] = \";\".join(cardinal_ner)\n",
        "  return buy_template\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nPtAoddJJWrA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sell_sm_test(new_sent, arguement_map, buy_template, doc):\n",
        "  ''' Extracting Buy(Buyer, Item, Price, Quantity, Source) template from the sentence \n",
        "      by running spacy's sm model.\n",
        "      This is called when selling event is triggered.\n",
        "    \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      arguement_map (dict): SRL obj arguement dictionary\n",
        "      buy_template (dict): buy template blank format which needs to be filled\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "    \n",
        "    Returns:\n",
        "      buy_template (dict): BUY template extracted\n",
        "  '''\n",
        "  buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "  \n",
        "  if \"ARG2\" in arguement_map:\n",
        "        \n",
        "    #extracting BUYER from ARG2 and SELLER from ARG0\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and ent.text in arguement_map[\"ARG2\"]:\n",
        "        buy_template[\"buyer\"] = ent.text\n",
        "      elif ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and \"ARG0\" in arguement_map and ent.text in arguement_map[\"ARG0\"]:\n",
        "        buy_template[\"source\"] = ent.text\n",
        "    \n",
        "    \n",
        "    #extracting ITEM sold from ARG1\n",
        "    item_list=[]\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"ORG\", \"PRODUCT\"] and \"ARG1\" in arguement_map and ent.text in arguement_map[\"ARG1\"]:\n",
        "        item_list.append(ent.text)\n",
        "    buy_template[\"item\"] = \";\".join(item_list)\n",
        "    \n",
        "    #extracting MONEY from ARG3 if available\n",
        "    for ent in doc.ents:\n",
        "      ent_tokenize = set(word_tokenize(ent.text))\n",
        "\n",
        "      if ent.label_ == \"MONEY\":\n",
        "        if \"ARG3\" in arguement_map:\n",
        "          arg3_tokenize = set(word_tokenize(arguement_map[\"ARG3\"]))\n",
        "          if len(ent_tokenize.intersection(arg3_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "        else:\n",
        "          arg1_tokenize = set(word_tokenize(arguement_map[\"ARG1\"]))\n",
        "          if len(ent_tokenize.intersection(arg1_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "    \n",
        "    #extracting QUANTITY if present by checking NER of Quantity or Cardinal\n",
        "    for ent in doc.ents:\n",
        "      quantity_ner = []\n",
        "      cardinal_ner = []\n",
        "      for ent in doc.ents:\n",
        "        if ent.label_ == \"QUANTITY\":\n",
        "          quantity_ner.append(ent.text)\n",
        "        elif ent.label_ == \"CARDINAL\":\n",
        "          cardinal_ner.append(ent.text)\n",
        "      \n",
        "      if len(quantity_ner) > 0:\n",
        "        buy_template[\"quantity\"] = \";\".join(quantity_ner)\n",
        "      else:\n",
        "        buy_template[\"quantity\"] = \";\".join(cardinal_ner)\n",
        "  return buy_template"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emrb-rezJwBK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sell_md_test(new_sent, arguement_map, buy_template, doc):\n",
        "  ''' Extracting Buy(Buyer, Item, Price, Quantity, Source) template from the sentence \n",
        "      by running spacy's md model.\n",
        "      This is called when selling event is triggered.\n",
        "    \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      arguement_map (dict): SRL obj arguement dictionary\n",
        "      buy_template (dict): buy template blank format which needs to be filled\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "    \n",
        "    Returns:\n",
        "      buy_template (dict): BUY template extracted\n",
        "  '''\n",
        "  buy_template = {\"buyer\": \"\", \"item\": \"\", \"price\": \"\", \"quantity\": \"\", \"source\": \"\"}\n",
        "  \n",
        "  if \"ARG2\" in arguement_map:\n",
        "        \n",
        "    #extracting BUYER from ARG2 and SELLER from ARG0\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and ent.text in arguement_map[\"ARG2\"]:\n",
        "        buy_template[\"buyer\"] = ent.text\n",
        "      elif ent.label_ in [\"PERSON\", \"NORP\", \"GPE\", \"ORG\"] and \"ARG0\" in arguement_map and ent.text in arguement_map[\"ARG0\"]:\n",
        "        buy_template[\"source\"] = ent.text\n",
        "    \n",
        "    \n",
        "    #extracting ITEM sold from ARG1\n",
        "    item_list=[]\n",
        "    for ent in doc.ents:\n",
        "      if ent.label_ in [\"PERSON\", \"NORP\", \"ORG\", \"PRODUCT\"] and \"ARG1\" in arguement_map and ent.text in arguement_map[\"ARG1\"]:\n",
        "        item_list.append(ent.text)\n",
        "    buy_template[\"item\"] = \";\".join(item_list)\n",
        "    \n",
        "    #extracting MONEY from ARG3 if available\n",
        "    for ent in doc.ents:\n",
        "      ent_tokenize = set(word_tokenize(ent.text))\n",
        "\n",
        "      if ent.label_ == \"MONEY\":\n",
        "        if \"ARG3\" in arguement_map:\n",
        "          arg3_tokenize = set(word_tokenize(arguement_map[\"ARG3\"]))\n",
        "          if len(ent_tokenize.intersection(arg3_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "        else:\n",
        "          arg1_tokenize = set(word_tokenize(arguement_map[\"ARG1\"]))\n",
        "          if len(ent_tokenize.intersection(arg1_tokenize)) == len(ent_tokenize):\n",
        "            buy_template[\"price\"] = ent.text\n",
        "    \n",
        "    #extracting QUANTITY if present by checking NER of Quantity or Cardinal\n",
        "    for ent in doc.ents:\n",
        "      quantity_ner = []\n",
        "      cardinal_ner = []\n",
        "      for ent in doc.ents:\n",
        "        if ent.label_ == \"QUANTITY\":\n",
        "          quantity_ner.append(ent.text)\n",
        "        elif ent.label_ == \"CARDINAL\":\n",
        "          cardinal_ner.append(ent.text)\n",
        "      \n",
        "      if len(quantity_ner) > 0:\n",
        "        buy_template[\"quantity\"] = \";\".join(quantity_ner)\n",
        "      else:\n",
        "        buy_template[\"quantity\"] = \";\".join(cardinal_ner)\n",
        "  return buy_template"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FRnj56UiXnk0",
        "colab_type": "text"
      },
      "source": [
        "# Work Template"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zqRxqIEJ4gmn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#list of all the stopwords\n",
        "stopwordSet=set(stopwords.words('english'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wnbugRyM4U4n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkWorkTrigger(new_sent, doc):\n",
        "  '''Check if the given sentence has any work template based on entities present \n",
        "    in the sentence. If Person, Organization and Job Title entity is present then \n",
        "    there's chance that it might have work template present so we will return all\n",
        "    entities related to Person, Organization and Job Title.\n",
        "    \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "\n",
        "    Returns:\n",
        "      person_entities (list): list of Person entities present in the sentence\n",
        "      title_entities (list): list of Job Title entities present in the sentence\n",
        "      organization_entities (list): list of organization entities present in the sentence\n",
        "      trigged: flag saying if the work template is triggered or not based on entities present\n",
        "  '''\n",
        "  person_entities = set()\n",
        "  organization_entities = set()\n",
        "  title_entities = set()\n",
        "  for ent in doc.ents:\n",
        "    if ent.label_ == \"PERSON\":\n",
        "      person_entities.add(ent.text)\n",
        "    elif ent.label_ == \"ORG\":\n",
        "      organization_entities.add(ent.text)\n",
        "    elif ent.label_ == \"TITLE\":\n",
        "      title_entities.add(ent.text)\n",
        "  if len(person_entities) > 0 or len(organization_entities) > 0 or len(title_entities) > 0:\n",
        "    trigged = True\n",
        "  else:\n",
        "    trigged = False\n",
        "  return person_entities, title_entities, organization_entities, trigged"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JWgGuWUrQ_b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def workTemplate(new_sent, link_coref_sent, srl_tags, doc1, EXTRACTION_LIST):\n",
        "  ''' Extracting WORK(Person, Organization, Position, Location) template from the sentence and storing\n",
        "    all the extracted templates into EXTRACTION_LIST\n",
        "    \n",
        "    Args:\n",
        "      new_sent (string): Sentence on which extraction is performed\n",
        "      link_coref_sent (string): Orignal set of sentences from where correference\n",
        "                                of current sentence is resolved.\n",
        "      srl_tags (object): SRL Tags obtained by performing AllenNlp SRL model on sentence\n",
        "      doc1 (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "      EXTRACTION_LIST (list): Previously extracted templates list to which new templates will be appended\n",
        "  '''\n",
        "  final_work_template_format = {\"template\": \"WORK\", \n",
        "                                \"sentences\": [link_coref_sent], \n",
        "                                \"arguements\": {\"1\": \"\",\n",
        "                                          \"2\": \"\",\n",
        "                                          \"3\": \"\",\n",
        "                                          \"4\": \"\"\n",
        "                                          }\n",
        "                              }\n",
        "  #check if the sentence has possiblity of work template\n",
        "  person_entities, title_entities, organization_entities, trigged = checkWorkTrigger(new_sent, doc1)\n",
        "\n",
        "  if not trigged:\n",
        "    return\n",
        "  \n",
        "  srl_verb_obj_list = srl_tags[\"verbs\"]\n",
        "  templates_extracted = {}\n",
        "  \n",
        "  #SRL verb \"became\" eg: Steve Jobs 'became' CEO of Apple.\n",
        "  work_SRL_verb_became(person_entities, title_entities, organization_entities, srl_verb_obj_list, templates_extracted)\n",
        "\n",
        "  #SRL verb \"worked\" eg: Steve Jobs 'worked' as CEO of Apple.\n",
        "  work_SRL_verb_worked(person_entities, title_entities, organization_entities, srl_verb_obj_list, templates_extracted)\n",
        "\n",
        "  #Dependency parser\n",
        "  work_dependency_parsing(doc1, person_entities, templates_extracted)\n",
        "  \n",
        "  #Phrase Matcher\n",
        "  work_phrase_matcher(doc1, templates_extracted)\n",
        "\n",
        "  for temp in templates_extracted.values():\n",
        "    final_work_template_format = {\"template\": \"WORK\", \n",
        "                                \"sentences\": [link_coref_sent], \n",
        "                                \"arguements\": {\"1\": \"\",\n",
        "                                          \"2\": \"\",\n",
        "                                          \"3\": \"\",\n",
        "                                          \"4\": \"\"\n",
        "                                          }\n",
        "                              }\n",
        "    \n",
        "    final_work_template_format[\"arguements\"][\"1\"] = temp[\"Person\"]\n",
        "    final_work_template_format[\"arguements\"][\"2\"] = temp[\"Organization\"]\n",
        "    final_work_template_format[\"arguements\"][\"3\"] = \";\".join(temp[\"Position\"])  #concatenate if person has multiple position at same org\n",
        "    final_work_template_format[\"arguements\"][\"4\"] = temp[\"Location\"]\n",
        "\n",
        "    EXTRACTION_LIST.append(final_work_template_format)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7cA4ytrbs5or",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def work_SRL_verb_became(person_entities, title_entities, organization_entities, srl_verb_obj_list, templates_extracted):\n",
        "  '''check if the ROOT verb is 'became' and the AGENT of the verb is Person, THEME is Organization \n",
        "    \n",
        "    Args:\n",
        "      person_entities (list): list of person entities\n",
        "      title_entities (list): list of title entities\n",
        "      organization_entities (list): list of organization entities\n",
        "      srl_verb_obj_list (list): srl verb object list\n",
        "      templates_extracted (list): previously extracted templates for WORK Template for given sentence\n",
        "  '''\n",
        "  for srl_verb_obj in srl_verb_obj_list:\n",
        "    #if the verb is 'became'\n",
        "    if srl_verb_obj[\"verb\"] == \"became\":\n",
        "        arguement_map = getSRLArguements(srl_verb_obj)\n",
        "        \n",
        "        if \"ARG1\" in arguement_map and \"ARG2\" in arguement_map:\n",
        "\n",
        "          work_template = {\"Person\": \"\", \"Organization\": \"\", \"Position\": \"\", \"Location\": \"\"}\n",
        "\n",
        "          #Looking for Person entity\n",
        "          for person in person_entities:\n",
        "            if person in arguement_map[\"ARG1\"]:\n",
        "              work_template[\"Person\"] = person\n",
        "\n",
        "          position_set = set()\n",
        "\n",
        "          #Looking for Title Entity\n",
        "          for title in title_entities:\n",
        "            if title in arguement_map[\"ARG2\"]:\n",
        "              position_set.add(title)\n",
        "          \n",
        "          if len(position_set) != 0:\n",
        "            work_template[\"Position\"] = position_set\n",
        "\n",
        "          #Looking for Organization Entity\n",
        "          for organization in organization_entities:\n",
        "            if organization in arguement_map[\"ARG2\"]:\n",
        "              work_template[\"Organization\"] = organization\n",
        "          \n",
        "          \n",
        "\n",
        "          if work_template[\"Person\"] != \"\" and work_template[\"Organization\"] != \"\":\n",
        "            hash = work_template[\"Person\"] + \"#\" + work_template[\"Organization\"]\n",
        "            if hash in template_extracted:\n",
        "              obj = template_extracted[hash]\n",
        "              obj[\"Position\"].update(work_template[\"Position\"])\n",
        "            else:\n",
        "              template_extracted[hash] = work_template"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dp9HpYN7xDh4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def work_SRL_verb_worked(person_entities, title_entities, organization_entities, srl_verb_obj_list, template_extracted):\n",
        "  '''check if the ROOT verb is 'worked' and the AGENT of the verb is Person, THEME is Organization \n",
        "    \n",
        "    Args:\n",
        "      person_entities (list): list of person entities\n",
        "      title_entities (list): list of title entities\n",
        "      organization_entities (list): list of organization entities\n",
        "      srl_verb_obj_list (list): srl verb object list\n",
        "      templates_extracted (list): previously extracted templates for WORK Template for given sentence\n",
        "  '''\n",
        "  for srl_verb_obj in srl_verb_obj_list:\n",
        "    if srl_verb_obj[\"verb\"] == \"worked\":\n",
        "      arguement_map = getSRLArguements(srl_verb_obj)\n",
        "      \n",
        "      #check only if AGENT is present that is Person entity working\n",
        "      if \"ARG0\" in arguement_map:\n",
        "        work_template = {\"Person\": \"\", \"Organization\": \"\", \"Position\": \"\", \"Location\": \"\"}\n",
        "\n",
        "        #Looking for Person Entity\n",
        "        for person in person_entities:\n",
        "          if person in arguement_map[\"ARG0\"]:\n",
        "            work_template[\"Person\"] = person\n",
        "\n",
        "        position_set = set()\n",
        "\n",
        "        #Looking for Title Entity\n",
        "        for title in title_entities:\n",
        "          if (\"ARG1\" in arguement_map and title in arguement_map[\"ARG1\"]) or (\"ARG2\" in arguement_map and title in arguement_map[\"ARG2\"]):\n",
        "            position_set.add(title)\n",
        "        \n",
        "        if len(position_set) != 0:\n",
        "          work_template[\"Position\"] = position_set\n",
        "\n",
        "        #Looking for Organization Entity\n",
        "        for organization in organization_entities:\n",
        "          if (\"ARG1\" in arguement_map and organization in arguement_map[\"ARG1\"]) or (\"ARG2\" in arguement_map and organization in arguement_map[\"ARG2\"]):\n",
        "            work_template[\"Organization\"] = organization\n",
        "        \n",
        "\n",
        "        if work_template[\"Person\"] != \"\" and work_template[\"Organization\"] != \"\":\n",
        "          hash = work_template[\"Person\"] + \"#\" + work_template[\"Organization\"]\n",
        "          if hash in template_extracted:\n",
        "            obj = template_extracted[hash]\n",
        "            obj[\"Position\"].update(work_template[\"Position\"])\n",
        "          else:\n",
        "            template_extracted[hash] = work_template"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5O7ubOVu8j_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def work_phrase_matcher(doc, template_extracted):\n",
        "  '''check if the sentence matches particular phrase like \n",
        "    'Apple co-founder Steve Jobs',\n",
        "    'Jeff Bezos, CEO of Amazon'\n",
        "    This uses spacy phrase matcher pipeline\n",
        "    \n",
        "    Args:\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "      templates_extracted (list): previously extracted templates for WORK Template for given sentence\n",
        "  '''\n",
        "  \n",
        "  for match_id, start, end in matcher(doc):\n",
        "    dict_work={\"Person\": \"\", \"Organization\": \"\", \"Position\": \"\", \"Location\": \"\"}\n",
        "    for ent in doc[start:end].ents:\n",
        "      if ent.label_ == \"PERSON\":\n",
        "        dict_work[\"Person\"]=ent.text\n",
        "      if ent.label_ == \"ORG\":\n",
        "        dict_work[\"Organization\"]=ent.text\n",
        "      if ent.label_ == \"TITLE\":\n",
        "        dict_work[\"Position\"]=ent\n",
        "    \n",
        "    hash = dict_work[\"Person\"] + \"#\" + dict_work[\"Organization\"]\n",
        "    if hash in template_extracted:\n",
        "      obj = template_extracted[hash]\n",
        "      obj[\"Position\"].add(dict_work[\"Position\"])\n",
        "    else:\n",
        "      template_extracted[hash] = dict_work"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yCd8zFIvwKl6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def work_dependency_parsing(doc, person_entities, template_extracted):\n",
        "  '''Applying dependency parsing technique, extracting the work template. Here we use\n",
        "    organization whose dependency parsing is pobj and from there we are tracing back to\n",
        "    positions  as well as the person entity using dependency parser.\n",
        "    \n",
        "    Args:\n",
        "      doc (spacy doc): doc obtained by passing spacy nlp pipeline\n",
        "      person_entities (list): list if person entity for current sentence\n",
        "      templates_extracted (list): previously extracted templates for WORK Template for given sentence\n",
        "  '''\n",
        "  company_index1 = 0\n",
        "  for token in doc:\n",
        "    template = {\"Person\": \"\", \"Position\": \"\", \"Organization\": \"\", \"Location\" : \"\"}\n",
        "    if token.dep_ == \"pobj\" and token.ent_type_ in [\"PERSON\",\"ORG\",\"GPE\",\"NORP\"] and token.head.dep_ == \"prep\" and token.head.head.ent_type_ == \"TITLE\":\n",
        "      \n",
        "      template[\"Organization\"] = token.text\n",
        "      current_company_index = token.i\n",
        "      position1 = token.head.head\n",
        "      positions_set = set([position1.text])\n",
        "\n",
        "      #using queue to find other conj and appos related to the position1\n",
        "      q = Queue()\n",
        "      for child in position1.conjuncts:\n",
        "        q.put(child)\n",
        "      \n",
        "      while not q.empty():\n",
        "        position = q.get()\n",
        "        if  position.ent_type_ == \"TITLE\" and position.i >= company_index1 and position.i <= current_company_index:\n",
        "          positions_set.add(position.text)\n",
        "          for child in position.children:\n",
        "            if child.dep_ == \"appos\":\n",
        "              q.put(child)\n",
        "      company_index1 = current_company_index\n",
        "      template[\"Position\"] = positions_set\n",
        "\n",
        "      if len(person_entities) == 1:\n",
        "        template[\"Person\"] = next(iter(person_entities))\n",
        "      else:\n",
        "        pflag=False\n",
        "        for ancestor in token.ancestors:\n",
        "          if ancestor.pos_ == \"AUX\" and not pflag:\n",
        "            for child in ancestor.children:\n",
        "              if child.dep_ == \"nsubj\":\n",
        "                template[\"Person\"] = child.text\n",
        "                pflag=True\n",
        "                break        \n",
        "    else:\n",
        "      if token.dep_ == \"pobj\" and token.ent_type_ in [\"PERSON\",\"ORG\",\"GPE\",\"NORP\"] and token.head.dep_ == \"prep\" and token.head.head.pos_ == \"VERB\":\n",
        "        template = {\"Person\": \"\", \"Position\": \"\", \"Organization\": \"\", \"Location\" : \"\"}\n",
        "\n",
        "        template[\"Organization\"] = token.text\n",
        "\n",
        "        current_company_index = token.i\n",
        "        verb = token.head.head\n",
        "\n",
        "        for verb_child in verb.children:\n",
        "          if verb_child.dep_ == \"prep\":\n",
        "            for verb_grand_child in verb_child.children:\n",
        "              if verb_grand_child.dep_ == \"pobj\" and verb_grand_child.ent_type_ == \"TITLE\":\n",
        "                position1 = verb_grand_child\n",
        "                positions_set = set([position1.text])\n",
        "\n",
        "                q = Queue()\n",
        "                for child in position1.conjuncts:\n",
        "                  q.put(child)\n",
        "        \n",
        "                while not q.empty():\n",
        "                  position = q.get()\n",
        "                  if  position.ent_type_ == \"TITLE\" and position.i >= company_index1 and position.i <= current_company_index:\n",
        "                    positions_set.add(position.text)\n",
        "\n",
        "                    for child in position.children:\n",
        "                      if child.dep_ == \"appos\":\n",
        "                        q.put(child)\n",
        "                template[\"Position\"] = positions_set\n",
        "\n",
        "                if len(person_entities) == 1:\n",
        "                  template[\"Person\"] = next(iter(person_entities))\n",
        "                else:\n",
        "                  pflag=False\n",
        "                  for ancestor in token.ancestors:\n",
        "                    if ancestor.pos_ == \"AUX\" and not pflag:\n",
        "                      for child in ancestor.children:\n",
        "                        if child.dep_ == \"nsubj\":\n",
        "                          template[\"Person\"] = child.text\n",
        "                          pflag=True\n",
        "                          break        \n",
        "        company_index1 = current_company_index\n",
        "\n",
        "    if template[\"Person\"] != \"\" and template[\"Organization\"] != \"\":\n",
        "      hash = template[\"Person\"] + \"#\" + template[\"Organization\"]\n",
        "      if hash in template_extracted:\n",
        "        obj = template_extracted[hash]\n",
        "        obj[\"Position\"].update(template[\"Position\"])\n",
        "      else:\n",
        "        template_extracted[hash] = template"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BtqLm1bWX7hS",
        "colab_type": "text"
      },
      "source": [
        "# Correference Resolution and  Template Extraction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VZhwfrVwuqr8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "import en_core_web_sm, en_core_web_md\n",
        "\n",
        "#Over here we will use en_core_web_sm model for all our task but some task which\n",
        "#requires increasing accuracy of the extraction we will use en_Core_web_md model\n",
        "nlp_other = en_core_web_md.load()\n",
        "nlp_all = en_core_web_sm.load()\n",
        "tokenizer = nlp_all.Defaults.create_tokenizer(nlp_all)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jtPKua8hn0kO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#adding sentencizer to nlp pipeline to segment paragraphs into sentences\n",
        "sent_pipe = nlp_other.create_pipe(\"sentencizer\")\n",
        "nlp_other.add_pipe(sent_pipe)\n",
        "\n",
        "#removing unnecessary pipes\n",
        "remove_pipes_other = [p for p in nlp_other.pipe_names if p not in [\"tagger\", \"sentencizer\", \"ner\"]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xb73nW88M0xj",
        "colab_type": "code",
        "outputId": "62c740a0-c3fa-406b-f13c-e9d82ec19a66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#Entity Ruler for Job Title which will eventually added to spacy pipeline\n",
        "ruler = EntityRuler(nlp_all)\n",
        "patterns = patternList(job_title)\n",
        "ruler.add_patterns(patterns)\n",
        "nlp_all.add_pipe(ruler)\n",
        "\n",
        "#disabling pipes for corref\n",
        "nlp_other.disable_pipes(remove_pipes_other)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('parser', <spacy.pipeline.pipes.DependencyParser at 0x7f4daf414a08>)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GRK8JMyvU4D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#nlp_all merge_entities pipe adding\n",
        "merge_pipe = nlp_all.create_pipe(\"merge_entities\")\n",
        "nlp_all.add_pipe(merge_pipe)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZghdWveAtvLL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Work Phrase Matcher Initialization\n",
        "matcher = PhraseMatcher(nlp_all.vocab, attr=\"ENT_TYPE\")\n",
        "matcher.add(\"Work_Temp\", None, nlp_all(\"Amazon's co-founder Jeff Bezos\"), nlp_all(\"Apple CEO Steve Jobs\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4OSeHDccY0eZ",
        "colab_type": "code",
        "outputId": "8aad7743-8138-493d-a81e-285fec635aee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377,
          "referenced_widgets": [
            "cb13f7f40a024af5873336daa3f3c6f9",
            "359482ceab754516a85d8f82a3ddc2f3",
            "370fa33a40fb4bf4919432b1e92cb29a",
            "9947f87aebc640c8ba86ea466333118a",
            "88ee5227f99c480d85a39af8ed83bdd8",
            "3f999763278b471597eea56c2a2db0bc",
            "fcc38b513ce94551a02f45288ca51330",
            "b9c1f209af644c4bbd838a08b3fae6cd",
            "dd7399da2c944392bf27bdd98de14da2",
            "396f585ad20044d1acc84d1558649585",
            "ee41aa48bf4d4ab5a5bba1ae9649a71c",
            "1df27c9a9d4243dd96232ba250df5841",
            "b224e0d4659a45c1b38a4f71540c6432",
            "b1cd11c562b04dabb98e44d81f9efeb6",
            "7a5caeda0c5a48aead8771116ad5528d",
            "cee6f845c0874f0f82cb85f0f05564fb",
            "c74c07c2d2ef40969d0417c1b4140be9",
            "f514a66213454115b86ae8867bbba6b2",
            "d6276bfbb537466b827e0cd4ada9ce1e",
            "85f0cf66d7da456fb4c3f8ada8c0c1b7",
            "247c518e543e4853854dff4ebdfaf44d",
            "bcdfd98b4bdb4ae8839097cb2ddf8979",
            "95abdb6b947944c4892c7f317e1b1390",
            "3535a5a3f9414362b290ec7b6c20347f",
            "8326689e5a334f8a8555ce223115090d",
            "656d2cbcd6704b4bb32efb6c7c6d8b27",
            "ddd553093fd6475ba5182d7f43ab8534",
            "c791108e80f84afab8fcd3dac0d1c119",
            "b9b554f7b2a94943bb5dcca7de8a4b7c",
            "0e804fb73b9d4238bbc524973e1736ec",
            "4d0b323f82db4b34844f23d010a11eea",
            "221ad883825d47ceae400b53407a9c6c",
            "fea0e9192f304cd8afa068c71c206133",
            "bf4cbe43e8bd432fadbd458ca102e292",
            "d35361602ac14debb6ef277324568d31",
            "3261ee26d59b442e9a70a31511af82dc",
            "2505316b5c914dc98719ead80735a0f7",
            "002675c246574e7384172d701f4737a4",
            "ca0a81d23a4b4be6b3c4830516c4d74a",
            "cd860c5a3110457fa4107fdd255a537a",
            "da7ad6d8b68a48e28266068afdcf38ce",
            "295394ec4afc4b519cce7ab5c4148841",
            "d41474395d7a4d3db1dc0036816e3409",
            "e6da87c9dbc84c3bb5efdaad63c6d3c8",
            "8aa0355ba33542a98b4abf3d31694ead",
            "e074239208c5463a8324a2600cea5e7c",
            "fb36f3220ed54e40b7a40f89fe257c9c",
            "cf0b7c632b224b299b4215cb10ae5854"
          ]
        }
      },
      "source": [
        "from allennlp.predictors.predictor import Predictor\n",
        "\n",
        "#Allennlp model for Correference Resolution\n",
        "from allennlp_models import coref\n",
        "predictor_coref = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/coref-spanbert-large-2020.02.27.tar.gz\")\n",
        "\n",
        "#Allennlp model for Semantic Role Labelling\n",
        "import allennlp_models.syntax.srl\n",
        "predictor_srl = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/bert-base-srl-2020.03.24.tar.gz\")\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1345947288/1345947288 [00:21<00:00, 64026329.32B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cb13f7f40a024af5873336daa3f3c6f9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=414.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dd7399da2c944392bf27bdd98de14da2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=665132540.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c74c07c2d2ef40969d0417c1b4140be9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Did not use initialization regex that was passed: _context_layer._module.weight_hh.*\n",
            "Did not use initialization regex that was passed: _context_layer._module.weight_ih.*\n",
            "100%|██████████| 406056588/406056588 [00:06<00:00, 61091905.03B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8326689e5a334f8a8555ce223115090d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fea0e9192f304cd8afa068c71c206133",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da7ad6d8b68a48e28266068afdcf38ce",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kufWVQyuvgKa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkNER(tokens, start_index, end_index, ent_para):\n",
        "  '''check whether any entity is present in token whose start index and end index \n",
        "    is given\n",
        "    \n",
        "    Args:\n",
        "      tokens (list): sentence token list\n",
        "      start_index (int): start index\n",
        "      end_index (int): end index\n",
        "      ent_para (list): list of entities in the given sentence\n",
        "\n",
        "    Returns:\n",
        "      flag (boolean): returns True if entity present, False otherwise\n",
        "  '''\n",
        "  aggregate = \" \".join(tokens[start_index: end_index])\n",
        "  if aggregate.strip() in ent_para:\n",
        "    return True\n",
        "  return False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j34gfMsLiCuV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def resolveClustersFromCoref(clusters, ent_para):\n",
        "  '''Intermediate step for resolving correference, \n",
        "    for each clusters it check whether it needs to replace the token with any entity,\n",
        "    based on that cluster_resolve_map is created which helps in resolving correference\n",
        "    at later stage\n",
        "\n",
        "    Args:\n",
        "      clusters (list of list): clusters formed by using allennlp correference resolution\n",
        "      ent_para (list): entities present in current paragraph\n",
        "\n",
        "    Returns:\n",
        "      cluster_resolve_map (dict): map for resolving cluster\n",
        "  '''\n",
        "  cluster_resolve_map = {}\n",
        "  for cluster in clusters:\n",
        "    orignal_start_index = cluster[0][0]\n",
        "    orignal_end_index = cluster[0][1] + 1\n",
        "    for i in range(1, len(cluster)):\n",
        "      start_index = cluster[i][0]\n",
        "      end_index = cluster[i][1] + 1\n",
        "      if checkNER(tokens, start_index, end_index, ent_para):\n",
        "        orignal_start_index = start_index\n",
        "        orignal_end_index = end_index\n",
        "      #map of :\n",
        "      #start_index_to_resolve -> ([start_index_to_resolve: end_index_to_resolve], resolved word, resolved word index)        \n",
        "      cluster_resolve_map[start_index] = ([start_index, end_index], \" \".join(tokens[orignal_start_index: orignal_end_index]), orignal_start_index)\n",
        "  return cluster_resolve_map"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lN2yCgBGW79_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def processSentence(orignalSent, sentDocs, sent_index, start_index, cluster_resolve_map, token_index_map, EXTRACTION_LIST):\n",
        "  '''This function is for processing the sentence. First, correference resolution is performed\n",
        "    which results in new sentence with resolved pronouns. Once correfenced sentence is obtained\n",
        "    we try to find templates from the sentence (PART, BUY and WORK Template)\n",
        "\n",
        "    Args:\n",
        "      orignalSent (string): Orignal sentence from the wikipedia document\n",
        "      sentDocs (spacy doc): doc obtained when ran on the sentence\n",
        "      sent_index (int): index of the current sentence in the paragraph\n",
        "      start_index (int): start index of the first token of current sentence relative to paragraph\n",
        "      cluster_resolve_map (dict): map for resolving clusters obtained by using allennlp coref\n",
        "      token_index_map (dict): map where each token index is related to corresponding sentence index\n",
        "      EXTRACTION_LIST (list): Previously extracted templates list to which new templates will be appended\n",
        "\n",
        "    Returns:\n",
        "      len (int): total tokens in given sentence\n",
        "  '''\n",
        "  newSent = []\n",
        "  backTrackSentIndex = sent_index\n",
        "\n",
        "  orignalTokens = []\n",
        "  for token in orignalSent:\n",
        "    orignalTokens.append(token.text)\n",
        "\n",
        "  #storing token_number -> sent_index\n",
        "  for i in range(0, len(orignalTokens)):\n",
        "    token_index_map[start_index + i] = sent_index\n",
        "  \n",
        "  i = 0\n",
        "  while(i < len(orignalTokens)):\n",
        "    #check if it needs resolution\n",
        "    if (start_index + i) in cluster_resolve_map:\n",
        "      tup = cluster_resolve_map[start_index + i]\n",
        "\n",
        "      #range to resolve\n",
        "      start = tup[0][0] - start_index\n",
        "      end = tup[0][1] - start_index\n",
        "      newSent.append(tup[1])\n",
        "      \n",
        "\n",
        "      #find resolved sent index\n",
        "      if tup[1] != orignalTokens[i]:\n",
        "        resolving_sent_index = token_index_map[tup[2]]\n",
        "        if resolving_sent_index < backTrackSentIndex:\n",
        "          backTrackSentIndex = resolving_sent_index\n",
        "      \n",
        "      i = end\n",
        "    else:\n",
        "      newSent.append(orignalTokens[i])\n",
        "      i = i + 1\n",
        "  link_coref_sent = \"\"\n",
        "  iter = itertools.islice(sentDocs.sents, backTrackSentIndex, sent_index + 1)\n",
        "  for iteration in iter:\n",
        "    link_coref_sent += \" \" + iteration.text\n",
        "\n",
        "  #replacing words like 'co - founder' -> 'co-founder'\n",
        "  new_sent =  TreebankWordDetokenizer().detokenize(newSent).replace(\" - \", \"-\")\n",
        "\n",
        "  #srl tags\n",
        "  srl_tags = predictor_srl.predict(sentence=new_sent)\n",
        "\n",
        "  #Applying spacy pipeline to correferenced sentence obtained\n",
        "  doc1 = nlp_all(new_sent)\n",
        "\n",
        "  # BUY TEMPLATE\n",
        "  buyTemplate(new_sent, link_coref_sent, srl_tags, doc1, EXTRACTION_LIST)\n",
        "\n",
        "  # WORK TEMPLATE\n",
        "  workTemplate(new_sent, link_coref_sent, srl_tags, doc1, EXTRACTION_LIST)\n",
        "\n",
        "  #LOCATION TEMPLATE\n",
        "  partTemplate(new_sent, link_coref_sent, doc1, EXTRACTION_LIST)\n",
        "  return len(orignalTokens)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eY8YecFyV5kr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#spliting wikipedia documents by paragraphs\n",
        "paras = content.split(\"\\n\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q-R8QEr3Y5hb",
        "colab_type": "code",
        "outputId": "664d22ca-33f8-48db-8f51-b1652ba5b437",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "EXTRACTION_LIST = []\n",
        "i = 0\n",
        "for para in tqdm(paras):\n",
        "  if para == \"\" or len(para) == 1 or para[-1] not in string.punctuation:\n",
        "    continue\n",
        "  #allennlp correference resoltion\n",
        "  corref = predictor_coref.predict(document=para)\n",
        "\n",
        "  sentDocs = nlp_other(para)\n",
        "  #get All Entities\n",
        "  ent_para = set([ent.text for ent in sentDocs.ents])\n",
        "  clusters = corref['clusters']\n",
        "\n",
        "  tokens = []\n",
        "  for token in sentDocs:\n",
        "    tokens.append(token.text)\n",
        "\n",
        "  #resolving clusters\n",
        "  cluster_resolve_map = resolveClustersFromCoref(clusters, ent_para)\n",
        "\n",
        "  #sentenizer to go over all the sentences\n",
        "  sent_index = 0\n",
        "  start_index = 0\n",
        "  token_index_map = {}\n",
        "  for sent in sentDocs.sents:\n",
        "    start_index += processSentence(sent, sentDocs, sent_index, start_index, cluster_resolve_map, token_index_map, EXTRACTION_LIST)\n",
        "    sent_index = sent_index + 1\n",
        "\n",
        "#extracted templates are outputted to json file  \n",
        "data = {\"document\" : filename, \"extraction\": EXTRACTION_LIST}\n",
        "output_f = open(output_filename, \"w+\", encoding=\"utf-8-sig\")\n",
        "json.dump(data, output_f)\n",
        "\n",
        "output_f.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 29/29 [01:11<00:00,  2.47s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rQda7_PwRN_L",
        "colab_type": "text"
      },
      "source": [
        "# Task1(Get Features)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_VkRLV_MSPGu",
        "colab_type": "code",
        "outputId": "94c6976a-de29-43ec-d473-f1770cad3469",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "from nltk.corpus import wordnet\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "from nltk.corpus import wordnet"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FEJwX2r6SaWZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_wn_feature(tokenSynset):\n",
        "  '''getting wordnet features from tokensynset\n",
        "\n",
        "    Args:\n",
        "      tokenSynset (list): list of token synsets \n",
        "\n",
        "    Returns:\n",
        "      synonyms\n",
        "      hypernymList\n",
        "      hyponymList\n",
        "      holonymList\n",
        "      meronymList\n",
        "      entailList\n",
        "  '''\n",
        "  synonyms=[]\n",
        "  hypernymList=[]\n",
        "  hyponymList=[]\n",
        "  holonymList=[]\n",
        "  meronymList=[]\n",
        "  entailList=[]\n",
        "  memberHolonymsList = []\n",
        "  for index, tempSynset in enumerate(tokenSynset):\n",
        "      \n",
        "      hypernyms = tempSynset.hypernyms()\n",
        "      for l in hypernyms:\n",
        "        hypernymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      hyponyms = tempSynset.hyponyms()\n",
        "      for l in hyponyms:\n",
        "        hyponymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      holonyms = tempSynset.part_holonyms()\n",
        "      for l in holonyms:\n",
        "        holonymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      meronyms = tempSynset.part_meronyms()\n",
        "      for l in meronyms:\n",
        "        meronymList.append(l.lemma_names()[0])\n",
        "        \n",
        "      entail = tempSynset.entailments()\n",
        "      for l in entail:\n",
        "        entailList.append(l.lemma_names()[0])\n",
        "        \n",
        "      for l in tempSynset.lemmas():\n",
        "        synonyms.append(l.name())\n",
        "      \n",
        "      memberHolonyms = tempSynset.member_holonyms()\n",
        "      for l in memberHolonyms:\n",
        "        memberHolonymsList.append(l.name())\n",
        "  return synonyms,hypernymList,hyponymList,holonymList,meronymList,entailList"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iMVIdT0vRXv-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_features_sent(doc):\n",
        "  '''Extracting features like wordnet features, pos tags, dependency, entity type\n",
        "    from the spacy doc of the sentence.\n",
        "    \n",
        "    Args:\n",
        "      doc (spacy doc): doc obtained from spacy pipeline on sentence\n",
        "    \n",
        "    Returns:\n",
        "      featureList (list): list of features for the sentence\n",
        "  '''\n",
        "  featureList={}\n",
        "  for token in doc:\n",
        "    if token.dep_ != 'punct' and token.dep_!='':\n",
        "      token_wn=token.text.replace(\" \",\"_\")\n",
        "      featureDict = {}\n",
        "      t_synset=wordnet.synsets(token_wn)\n",
        "      synonyms_List,hypernym_List,hyponym_List,holonym_List,meronym_List,entail_List = get_wn_feature(t_synset)\n",
        "      featureDict[\"synonyms\"]=synonyms_List\n",
        "      featureDict[\"hypernyms\"]=hypernym_List\n",
        "      featureDict[\"hyponyms\"]=hyponym_List\n",
        "      featureDict[\"holonyms\"]=holonym_List\n",
        "      featureDict[\"meronyms\"]=meronym_List\n",
        "      featureDict[\"entailments\"]=entail_List\n",
        "      featureDict[\"token\"] = token\n",
        "      featureDict[\"lemmaVal\"] = token.lemma\n",
        "      featureDict[\"lemma\"] = token.lemma_\n",
        "      featureDict[\"pos\"] = token.pos_\n",
        "      featureDict[\"tag\"] = token.tag_\n",
        "      featureDict[\"shape\"] = token.shape_\n",
        "      featureDict[\"isAlpha\"] = token.is_alpha\n",
        "      featureDict[\"isStop\"] = token.is_stop\n",
        "      featureDict[\"dependency\"]=token.dep_\n",
        "      featureDict[\"headText\"] = token.head.text\n",
        "      featureDict[\"headTag\"]=token.head.pos_\n",
        "      featureDict[\"entity_rel\"]=token.ent_type_\n",
        "      featureDict[\"ent_type\"]=token.ent_id\n",
        "      featureDict[\"children\"] = [child for child in token.children]\n",
        "      featureList[token.text]=featureDict\n",
        "  return featureList"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcPTg-BRReeL",
        "colab_type": "code",
        "outputId": "c6e3ffdd-74eb-4710-d38f-43aa79df5318",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "doc=nlp_all(\"Powell Jobs is a founding member of the Climate Leadership Council.\")\n",
        "print(get_features_sent(doc))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'Powell Jobs': {'synonyms': [], 'hypernyms': [], 'hyponyms': [], 'holonyms': [], 'meronyms': [], 'entailments': [], 'token': Powell Jobs, 'lemmaVal': 17815689719766628822, 'lemma': 'Powell Jobs', 'pos': 'PROPN', 'tag': 'NNP', 'shape': 'Xxxxx Xxxx', 'isAlpha': False, 'isStop': False, 'dependency': 'nsubj', 'headText': 'is', 'headTag': 'AUX', 'entity_rel': 'PERSON', 'ent_type': 0, 'children': []}, 'is': {'synonyms': ['be', 'be', 'be', 'exist', 'be', 'be', 'equal', 'be', 'constitute', 'represent', 'make_up', 'comprise', 'be', 'be', 'follow', 'embody', 'be', 'personify', 'be', 'be', 'live', 'be', 'cost', 'be'], 'hypernyms': ['typify', 'take', 'stay', 'be'], 'hyponyms': ['abound', 'accept', 'account', 'account_for', 'act', 'answer', 'appear', 'bake', 'balance', 'be_well', 'beat', 'begin', 'begin', 'belong', 'belong', 'belong', 'belong', 'breathe', 'buy', 'clean', 'cohere', 'come_in_for', 'come_in_handy', 'compact', 'compare', 'confuse', 'connect', 'consist', 'consist', 'contain', 'contain', 'continue', 'cost', 'count', 'count', 'cover', 'cut', 'cut_across', 'deck', 'depend', 'deserve', 'disagree', 'distribute', 'diverge', 'draw', 'end', 'fall', 'fall', 'feel', 'figure', 'fit', 'gape', 'go', 'gravitate', 'hail', 'hang', 'head', 'hold', 'hoodoo', 'hum', 'impend', 'incarnate', 'iridesce', 'jumble', 'kill', 'lend', 'let_go', 'lie', 'litter', 'loiter', 'look', 'look', 'lubricate', 'make', 'make_sense', 'measure', 'mope', 'object', 'osculate', 'owe', 'pay', 'point', 'press', 'promise', 'prove', 'put_out', 'rage', 'range', 'rank', 'rate', 'recognize', 'relate', 'remain', 'represent', 'rest', 'retard', 'run', 'run_into', 'rut', 'seem', 'seethe', 'sell', 'sell', 'sell', 'shine', 'shine', 'sparkle', 'specify', 'squat', 'stagnate', 'stagnate', 'stand', 'stand', 'stand_by', 'stay', 'stay', 'stick', 'stink', 'subtend', 'suck', 'suffer', 'suffer', 'suit', 'swim', 'swim', 'swing', 'tend', 'test', 'total', 'translate', 'transplant', 'trim', 'underlie', 'want', 'wash', 'wind', 'work', 'attend', 'belong', 'center_on', 'come', 'cover', 'extend', 'face', 'follow', 'go', 'inhabit', 'lie', 'lie', 'occupy', 'populate', 'reach', 'run', 'sit', 'sit', 'stand_back', 'straddle', 'stretch', 'coexist', 'come', 'distribute', 'dwell', 'dwell', 'endanger', 'flow', 'indwell', 'kick_around', 'preexist', 'prevail', 'equate', 'match', 'represent', 'translate', 'compose', 'fall_into', 'form', 'make', 'present', 'range', 'supplement', 'cox', 'vet', 'body', 'exemplify', 'set_back'], 'holonyms': [], 'meronyms': [], 'entailments': [], 'token': is, 'lemmaVal': 10382539506755952630, 'lemma': 'be', 'pos': 'AUX', 'tag': 'VBZ', 'shape': 'xx', 'isAlpha': True, 'isStop': True, 'dependency': 'ROOT', 'headText': 'is', 'headTag': 'AUX', 'entity_rel': '', 'ent_type': 0, 'children': [Powell Jobs, member, .]}, 'a': {'synonyms': ['angstrom', 'angstrom_unit', 'A', 'vitamin_A', 'antiophthalmic_factor', 'axerophthol', 'A', 'deoxyadenosine_monophosphate', 'A', 'adenine', 'A', 'ampere', 'amp', 'A', 'A', 'a', 'A', 'type_A', 'group_A'], 'hypernyms': ['metric_linear_unit', 'fat-soluble_vitamin', 'nucleotide', 'purine', 'current_unit', 'letter', 'blood_group'], 'hyponyms': ['vitamin_A1', 'vitamin_A2'], 'holonyms': ['nanometer', 'abampere'], 'meronyms': ['picometer', 'milliampere'], 'entailments': [], 'token': a, 'lemmaVal': 11901859001352538922, 'lemma': 'a', 'pos': 'DET', 'tag': 'DT', 'shape': 'x', 'isAlpha': True, 'isStop': True, 'dependency': 'det', 'headText': 'member', 'headTag': 'NOUN', 'entity_rel': '', 'ent_type': 0, 'children': []}, 'founding': {'synonyms': ['initiation', 'founding', 'foundation', 'institution', 'origination', 'creation', 'innovation', 'introduction', 'instauration', 'establish', 'set_up', 'found', 'launch', 'establish', 'found', 'plant', 'constitute', 'institute', 'establish', 'base', 'ground', 'found'], 'hypernyms': ['beginning', 'open', 'initiate'], 'hyponyms': ['authorship', 'appoint', 'fix', 'build'], 'holonyms': [], 'meronyms': [], 'entailments': [], 'token': founding, 'lemmaVal': 602335199165703938, 'lemma': 'found', 'pos': 'VERB', 'tag': 'VBG', 'shape': 'xxxx', 'isAlpha': True, 'isStop': False, 'dependency': 'amod', 'headText': 'member', 'headTag': 'NOUN', 'entity_rel': '', 'ent_type': 0, 'children': []}, 'member': {'synonyms': ['member', 'fellow_member', 'member', 'extremity', 'appendage', 'member', 'member', 'penis', 'phallus', 'member'], 'hypernyms': ['associate', 'part', 'external_body_part', 'unit', 'erectile_organ'], 'hyponyms': ['Areopagite', 'board_member', 'brother', 'cabalist', 'charter_member', 'clansman', 'club_member', 'commissioner', 'committee_member', 'Conservative', 'council_member', 'fellow', 'homeboy', 'homegirl', 'huddler', 'inductee', 'joiner', 'kibbutznik', 'kolkhoznik', 'pledge', 'Rosicrucian', 'Rosicrucian', 'Rotarian', 'sister', 'sodalist', 'tribesman', 'chelicera', 'claw', 'digit', 'fang', 'fin', 'limb', 'mouthpart', 'parapodium', 'swimmeret', 'cock', 'micropenis'], 'holonyms': ['male_genitalia', 'male_reproductive_system'], 'meronyms': ['glans_penis', 'prepuce', 'urethra', 'vena_bulbi_penis'], 'entailments': [], 'token': member, 'lemmaVal': 14721843519903598875, 'lemma': 'member', 'pos': 'NOUN', 'tag': 'NN', 'shape': 'xxxx', 'isAlpha': True, 'isStop': False, 'dependency': 'attr', 'headText': 'is', 'headTag': 'AUX', 'entity_rel': '', 'ent_type': 0, 'children': [a, founding, of]}, 'of': {'synonyms': [], 'hypernyms': [], 'hyponyms': [], 'holonyms': [], 'meronyms': [], 'entailments': [], 'token': of, 'lemmaVal': 886050111519832510, 'lemma': 'of', 'pos': 'ADP', 'tag': 'IN', 'shape': 'xx', 'isAlpha': True, 'isStop': True, 'dependency': 'prep', 'headText': 'member', 'headTag': 'NOUN', 'entity_rel': '', 'ent_type': 0, 'children': [the Climate Leadership Council]}, 'the Climate Leadership Council': {'synonyms': [], 'hypernyms': [], 'hyponyms': [], 'holonyms': [], 'meronyms': [], 'entailments': [], 'token': the Climate Leadership Council, 'lemmaVal': 8651452168038286723, 'lemma': 'the Climate Leadership Council', 'pos': 'PROPN', 'tag': 'NNP', 'shape': 'xxx Xxxxx Xxxxx Xxxxx', 'isAlpha': False, 'isStop': False, 'dependency': 'pobj', 'headText': 'of', 'headTag': 'ADP', 'entity_rel': 'ORG', 'ent_type': 0, 'children': []}}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lubjtTj1LtCX",
        "colab_type": "text"
      },
      "source": [
        "# ROUGH WORK"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWjuZOmnJ4Af",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpe = [] # countries, cities, states\n",
        "loc = [] # non gpe locations, mountain ranges, bodies of water\n",
        "\n",
        "\n",
        "doc = nlp(\"Richardson is principal city in Dallas which is in United States.\")\n",
        "for ent in doc.ents:\n",
        "    if (ent.label_ == 'GPE'):\n",
        "        gpe.append(ent.text)\n",
        "    elif (ent.label_ == 'LOC'):\n",
        "        loc.append(ent.text)\n",
        "\n",
        "print(gpe)\n",
        "print(loc)\n",
        "cities = []\n",
        "countries = []\n",
        "other_places = []\n",
        "import wikipedia\n",
        "for text in gpe:\n",
        "    summary = str(wikipedia.summary(text))\n",
        "    # print(summary)\n",
        "    if ('city' in summary):\n",
        "        cities.append(text)\n",
        "    elif ('country' in summary):\n",
        "        countries.append(text)\n",
        "    else:\n",
        "        other_places.append(text)\n",
        "\n",
        "for text in loc:\n",
        "    other_places.append(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ie6RF28OKWa6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cities"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_0nElQ5Kkpu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "countries"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "29QNsu7ZKmSy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install pycountry"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MuAHCY3PQVNp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pycountry\n",
        "text = \"U.S (New York), United Kingdom (London)\"\n",
        "for country in pycountry.countries:\n",
        "    if country.name in text:\n",
        "        print(country.name)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QBNgCd0VQb8c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install geotext\n",
        "from geotext import GeoText"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7RDetzJ3Rsi_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpe = [] # countries, cities, states\n",
        "loc = [] # non gpe locations, mountain ranges, bodies of water\n",
        "\n",
        "\n",
        "doc = nlp(\"Richardson is principal city in Dallas which is in United States.\")\n",
        "for ent in doc.ents:\n",
        "    if (ent.label_ == 'GPE'):\n",
        "        gpe.append(ent.text)\n",
        "    elif (ent.label_ == 'LOC'):\n",
        "        loc.append(ent.text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzOnn4phR6lv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "places = GeoText(\"Richardson is principal city in Dallas which is in U.S.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2V41hvErSBb3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "cities = list(places.cities)\n",
        "cities\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qfjUXcraSCau",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "places.countries"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2b7XQaxJS5vJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "places.nationalities"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kcepThlCq2Of",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    }
  ]
}