Topic-modelling-NMF-freedom-June18.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "87c2a686-e950-4237-a66b-b638e9f02255",
   "metadata": {},
   "source": [
    "# NMF at message level\n",
    "1. split messages further into sentences (include commas)\n",
    "2. add an additional `message_idx` (for aggregating later)\n",
    "3. split df into 1 word and multi-word\n",
    "    - ignore 1 word df for now\n",
    "4. run nmf\n",
    "5. aggregate at conv level\n",
    "    - sort by `[\"conv_id\", \"msg_idx\"]`\n",
    "    - group by `conv_id`: concat text, concat topics, concat topic keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6fab928b-68ff-4911-944b-f996fbcfd6cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "from pathlib import Path\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.decomposition import NMF\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from joblib import dump\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "from collections import Counter\n",
    "from operator import itemgetter\n",
    "from gensim.models.nmf import Nmf\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('darkgrid')\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from NMF_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "28c2e9fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set values for rest of script\n",
    "best_num_topics = 200 \n",
    "top_k = 3 # k for top_k topics\n",
    "n_top_words = 5 # n for n_top_words (number of words for topic key words)\n",
    "docweights_threshold = 0.1 # threshold for considering a topic is relevant in NMF\n",
    "save_folder = \"june21\"\n",
    "Path(f\"./nmf_topics/{save_folder}\").mkdir(parents=True, exist_ok=True)\n",
    "Path(f\"./nmf_models/{save_folder}\").mkdir(parents=True, exist_ok=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "277e2e4c-aa0e-4893-b8e2-df0f0abf3263",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"freedom_intent_no_escalate.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "14b4c2b9-1588-4623-8f68-7a39790802ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>CLUSTER_ID</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>SESSION_ID</th>\n",
       "      <th>CHANNEL</th>\n",
       "      <th>MESSAGE_INTENT_1</th>\n",
       "      <th>MESSAGE_INTENT_CONFIDENCE_1</th>\n",
       "      <th>MESSAGE_INTENT_2</th>\n",
       "      <th>MESSAGE_INTENT_CONFIDENCE_2</th>\n",
       "      <th>MESSAGE_INTENT_3</th>\n",
       "      <th>MESSAGE_INTENT_CONFIDENCE_3</th>\n",
       "      <th>MESSAGE_INTENT</th>\n",
       "      <th>MESSAGE_CONFIDENCE</th>\n",
       "      <th>IF_VALID_INTENT</th>\n",
       "      <th>LANGUAGE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>netel</td>\n",
       "      <td>d63b71e0-e4c7-4012-822b-135f884dd90d:::4</td>\n",
       "      <td>BOT</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>$shop_accessory_misc</td>\n",
       "      <td>0.953275</td>\n",
       "      <td>True</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>netel</td>\n",
       "      <td>5241a58a-9b9a-422e-b451-3832ac96e1dc:::6</td>\n",
       "      <td>BOT</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>$shop_accessory_misc</td>\n",
       "      <td>0.953275</td>\n",
       "      <td>True</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>netel</td>\n",
       "      <td>b864d516-66b3-4223-a852-e4b428b7578d:::4</td>\n",
       "      <td>BOT</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>$shop_accessory_misc</td>\n",
       "      <td>0.953275</td>\n",
       "      <td>True</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>netel</td>\n",
       "      <td>719801fc-34c6-4ff9-8d7b-bb32d4de6766:::4</td>\n",
       "      <td>BOT</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>$shop_accessory_misc</td>\n",
       "      <td>0.953275</td>\n",
       "      <td>True</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>netel</td>\n",
       "      <td>3808e306-4e23-4292-94f3-d1fd1b9bfd33:::4</td>\n",
       "      <td>BOT</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>$shop_accessory_misc</td>\n",
       "      <td>0.953275</td>\n",
       "      <td>True</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  CLUSTER_ID MESSAGE                                SESSION_ID  \\\n",
       "0           2           0   netel  d63b71e0-e4c7-4012-822b-135f884dd90d:::4   \n",
       "1           3           0   netel  5241a58a-9b9a-422e-b451-3832ac96e1dc:::6   \n",
       "2          11           0   netel  b864d516-66b3-4223-a852-e4b428b7578d:::4   \n",
       "3          15           0   netel  719801fc-34c6-4ff9-8d7b-bb32d4de6766:::4   \n",
       "4          22           0   netel  3808e306-4e23-4292-94f3-d1fd1b9bfd33:::4   \n",
       "\n",
       "  CHANNEL  MESSAGE_INTENT_1  MESSAGE_INTENT_CONFIDENCE_1  MESSAGE_INTENT_2  \\\n",
       "0     BOT                -1                           -1                -1   \n",
       "1     BOT                -1                           -1                -1   \n",
       "2     BOT                -1                           -1                -1   \n",
       "3     BOT                -1                           -1                -1   \n",
       "4     BOT                -1                           -1                -1   \n",
       "\n",
       "   MESSAGE_INTENT_CONFIDENCE_2  MESSAGE_INTENT_3  MESSAGE_INTENT_CONFIDENCE_3  \\\n",
       "0                           -1                -1                           -1   \n",
       "1                           -1                -1                           -1   \n",
       "2                           -1                -1                           -1   \n",
       "3                           -1                -1                           -1   \n",
       "4                           -1                -1                           -1   \n",
       "\n",
       "         MESSAGE_INTENT  MESSAGE_CONFIDENCE  IF_VALID_INTENT LANGUAGE  \n",
       "0  $shop_accessory_misc            0.953275             True       en  \n",
       "1  $shop_accessory_misc            0.953275             True       en  \n",
       "2  $shop_accessory_misc            0.953275             True       en  \n",
       "3  $shop_accessory_misc            0.953275             True       en  \n",
       "4  $shop_accessory_misc            0.953275             True       en  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80615f71-06e1-4cda-82e9-7f9114525568",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2f3b6aad-c4aa-4e37-ad75-f93886c5bbc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d63b71e0-e4c7-4012-822b-135f884dd90d</td>\n",
       "      <td>4</td>\n",
       "      <td>netel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5241a58a-9b9a-422e-b451-3832ac96e1dc</td>\n",
       "      <td>6</td>\n",
       "      <td>netel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b864d516-66b3-4223-a852-e4b428b7578d</td>\n",
       "      <td>4</td>\n",
       "      <td>netel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>719801fc-34c6-4ff9-8d7b-bb32d4de6766</td>\n",
       "      <td>4</td>\n",
       "      <td>netel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3808e306-4e23-4292-94f3-d1fd1b9bfd33</td>\n",
       "      <td>4</td>\n",
       "      <td>netel</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        conversation_id  message_id MESSAGE\n",
       "0  d63b71e0-e4c7-4012-822b-135f884dd90d           4   netel\n",
       "1  5241a58a-9b9a-422e-b451-3832ac96e1dc           6   netel\n",
       "2  b864d516-66b3-4223-a852-e4b428b7578d           4   netel\n",
       "3  719801fc-34c6-4ff9-8d7b-bb32d4de6766           4   netel\n",
       "4  3808e306-4e23-4292-94f3-d1fd1b9bfd33           4   netel"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[~df.MESSAGE.str.contains(\"\"\"{ \" com \"\"\")]\n",
    "# add conv id and message id to sort by\n",
    "df[\"conversation_id\"] = df[\"SESSION_ID\"].str.split(\":::\").str[0]\n",
    "df[\"message_id\"] = df[\"SESSION_ID\"].str.split(\":::\").str[1].astype(int)\n",
    "# sort to make grouping easier\n",
    "# df = df.sort_values(by=[\"conversation_id\", \"message_id\"])\n",
    "df = df[[\"conversation_id\", \"message_id\", \"MESSAGE\"]]\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08c7711f-8d07-41b7-9a40-440cdf887416",
   "metadata": {},
   "source": [
    "split the messages further into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2057ac4c-82ab-4cd5-884e-5b3f1f2fe124",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3525</th>\n",
       "      <td>8087c72a-3246-42ee-ac83-db6d87be6bfc</td>\n",
       "      <td>1</td>\n",
       "      <td>how can i do that</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3525</th>\n",
       "      <td>8087c72a-3246-42ee-ac83-db6d87be6bfc</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3526</th>\n",
       "      <td>e6c37c6b-52c8-4ed0-a02d-ec77d96ad839</td>\n",
       "      <td>3</td>\n",
       "      <td>is there any deal to buy apple watch 6 in mont...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3527</th>\n",
       "      <td>13d88447-fdc4-42ac-87d9-9bb1641a8062</td>\n",
       "      <td>1</td>\n",
       "      <td>hi i recently transferred my phone number from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3527</th>\n",
       "      <td>13d88447-fdc4-42ac-87d9-9bb1641a8062</td>\n",
       "      <td>1</td>\n",
       "      <td>i am unable to access it due to the fact that...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3527</th>\n",
       "      <td>13d88447-fdc4-42ac-87d9-9bb1641a8062</td>\n",
       "      <td>1</td>\n",
       "      <td>the account name is under neper</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3527</th>\n",
       "      <td>13d88447-fdc4-42ac-87d9-9bb1641a8062</td>\n",
       "      <td>1</td>\n",
       "      <td>i am looking to cancel the plan on the accoun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3527</th>\n",
       "      <td>13d88447-fdc4-42ac-87d9-9bb1641a8062</td>\n",
       "      <td>1</td>\n",
       "      <td>thank you</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3528</th>\n",
       "      <td>3836504886472679</td>\n",
       "      <td>18</td>\n",
       "      <td>have a great night</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3529</th>\n",
       "      <td>3473478809418490</td>\n",
       "      <td>5</td>\n",
       "      <td>i topped up my account online but i got nothin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           conversation_id  message_id  \\\n",
       "3525  8087c72a-3246-42ee-ac83-db6d87be6bfc           1   \n",
       "3525  8087c72a-3246-42ee-ac83-db6d87be6bfc           1   \n",
       "3526  e6c37c6b-52c8-4ed0-a02d-ec77d96ad839           3   \n",
       "3527  13d88447-fdc4-42ac-87d9-9bb1641a8062           1   \n",
       "3527  13d88447-fdc4-42ac-87d9-9bb1641a8062           1   \n",
       "3527  13d88447-fdc4-42ac-87d9-9bb1641a8062           1   \n",
       "3527  13d88447-fdc4-42ac-87d9-9bb1641a8062           1   \n",
       "3527  13d88447-fdc4-42ac-87d9-9bb1641a8062           1   \n",
       "3528                      3836504886472679          18   \n",
       "3529                      3473478809418490           5   \n",
       "\n",
       "                                                MESSAGE  \n",
       "3525                                  how can i do that  \n",
       "3525                                                     \n",
       "3526  is there any deal to buy apple watch 6 in mont...  \n",
       "3527  hi i recently transferred my phone number from...  \n",
       "3527   i am unable to access it due to the fact that...  \n",
       "3527                   the account name is under neper   \n",
       "3527   i am looking to cancel the plan on the accoun...  \n",
       "3527                                          thank you  \n",
       "3528                                 have a great night  \n",
       "3529  i topped up my account online but i got nothin...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the messages further into sentences\n",
    "df = df.drop('MESSAGE', axis=1).join(df['MESSAGE'].str.split(r\"[,.!?]\", expand=True).stack().reset_index(level=1, drop=True).rename('MESSAGE'))\n",
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9aa9aa4c-0502-45ed-a2eb-d5d56a34b426",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>session_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>20</td>\n",
       "      <td>data + talk</td>\n",
       "      <td>3</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>22</td>\n",
       "      <td>more than 10 gb</td>\n",
       "      <td>4</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>24</td>\n",
       "      <td>15 gb freedom data</td>\n",
       "      <td>5</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>ffb30da4-4274-4dde-a755-eb5160834e8c</td>\n",
       "      <td>4</td>\n",
       "      <td>netel</td>\n",
       "      <td>0</td>\n",
       "      <td>ffb30da4-4274-4dde-a755-eb5160834e8c_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>ffb30da4-4274-4dde-a755-eb5160834e8c</td>\n",
       "      <td>6</td>\n",
       "      <td>netel</td>\n",
       "      <td>1</td>\n",
       "      <td>ffb30da4-4274-4dde-a755-eb5160834e8c_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td>hello</td>\n",
       "      <td>0</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td>i purchased the cannes outdoor sofa however i...</td>\n",
       "      <td>1</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td>are they machine washable</td>\n",
       "      <td>2</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>5</td>\n",
       "      <td>message with a rep</td>\n",
       "      <td>4</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           conversation_id  message_id  \\\n",
       "883   ff9ac61f-8f98-4516-9342-7aa1177514d9          20   \n",
       "1062  ff9ac61f-8f98-4516-9342-7aa1177514d9          22   \n",
       "1149  ff9ac61f-8f98-4516-9342-7aa1177514d9          24   \n",
       "73    ffb30da4-4274-4dde-a755-eb5160834e8c           4   \n",
       "39    ffb30da4-4274-4dde-a755-eb5160834e8c           6   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "247   ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           5   \n",
       "\n",
       "                                                MESSAGE  sentence_id  \\\n",
       "883                                         data + talk            3   \n",
       "1062                                    more than 10 gb            4   \n",
       "1149                                 15 gb freedom data            5   \n",
       "73                                                netel            0   \n",
       "39                                                netel            1   \n",
       "3049                                              hello            0   \n",
       "3049   i purchased the cannes outdoor sofa however i...            1   \n",
       "3049                          are they machine washable            2   \n",
       "3049                                                               3   \n",
       "247                                  message with a rep            4   \n",
       "\n",
       "                                  session_id  \n",
       "883   ff9ac61f-8f98-4516-9342-7aa1177514d9_3  \n",
       "1062  ff9ac61f-8f98-4516-9342-7aa1177514d9_4  \n",
       "1149  ff9ac61f-8f98-4516-9342-7aa1177514d9_5  \n",
       "73    ffb30da4-4274-4dde-a755-eb5160834e8c_0  \n",
       "39    ffb30da4-4274-4dde-a755-eb5160834e8c_1  \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_0  \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_1  \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_2  \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_3  \n",
       "247   ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_4  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add sentence_id to track message order for aggregation\n",
    "# sort to make grouping easier\n",
    "df = df.sort_values(by=[\"conversation_id\", \"message_id\"])\n",
    "df[\"sentence_id\"] = df.groupby(\"conversation_id\").cumcount()\n",
    "df[\"session_id\"] = df[\"conversation_id\"]+\"_\"+df[\"sentence_id\"].astype(str)\n",
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "08a2c22f-9019-47d3-a005-03a2ebfd542b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3496, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1240</th>\n",
       "      <td>ff097a90-5972-46d1-845a-3c9a399b70bc</td>\n",
       "      <td>1</td>\n",
       "      <td>i need help</td>\n",
       "      <td>0</td>\n",
       "      <td>ff097a90-5972-46d1-845a-3c9a399b70bc_0</td>\n",
       "      <td>3</td>\n",
       "      <td>[need, help]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1883</th>\n",
       "      <td>ff93a689-8d1a-49c2-a5bc-54dd321707a3</td>\n",
       "      <td>1</td>\n",
       "      <td>i am getting a phone for my son and looking to...</td>\n",
       "      <td>0</td>\n",
       "      <td>ff93a689-8d1a-49c2-a5bc-54dd321707a3_0</td>\n",
       "      <td>26</td>\n",
       "      <td>[get, phone, son, look, plan, iphon, se, packag]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>7</td>\n",
       "      <td>data + talk</td>\n",
       "      <td>0</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_0</td>\n",
       "      <td>3</td>\n",
       "      <td>[data, talk]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>9</td>\n",
       "      <td>less than 10 gb</td>\n",
       "      <td>1</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_1</td>\n",
       "      <td>4</td>\n",
       "      <td>[gb]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1284</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>14</td>\n",
       "      <td>10 gb</td>\n",
       "      <td>2</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_2</td>\n",
       "      <td>2</td>\n",
       "      <td>[gb]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>20</td>\n",
       "      <td>data + talk</td>\n",
       "      <td>3</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_3</td>\n",
       "      <td>3</td>\n",
       "      <td>[data, talk]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>22</td>\n",
       "      <td>more than 10 gb</td>\n",
       "      <td>4</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_4</td>\n",
       "      <td>4</td>\n",
       "      <td>[gb]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149</th>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9</td>\n",
       "      <td>24</td>\n",
       "      <td>15 gb freedom data</td>\n",
       "      <td>5</td>\n",
       "      <td>ff9ac61f-8f98-4516-9342-7aa1177514d9_5</td>\n",
       "      <td>4</td>\n",
       "      <td>[gb, freedom, data]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td>i purchased the cannes outdoor sofa however i...</td>\n",
       "      <td>1</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_1</td>\n",
       "      <td>17</td>\n",
       "      <td>[purchas, cann, outdoor, sofa, care, instruct,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3049</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>2</td>\n",
       "      <td>are they macne washable</td>\n",
       "      <td>2</td>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_2</td>\n",
       "      <td>5</td>\n",
       "      <td>[macn, washabl]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           conversation_id  message_id  \\\n",
       "1240  ff097a90-5972-46d1-845a-3c9a399b70bc           1   \n",
       "1883  ff93a689-8d1a-49c2-a5bc-54dd321707a3           1   \n",
       "859   ff9ac61f-8f98-4516-9342-7aa1177514d9           7   \n",
       "993   ff9ac61f-8f98-4516-9342-7aa1177514d9           9   \n",
       "1284  ff9ac61f-8f98-4516-9342-7aa1177514d9          14   \n",
       "883   ff9ac61f-8f98-4516-9342-7aa1177514d9          20   \n",
       "1062  ff9ac61f-8f98-4516-9342-7aa1177514d9          22   \n",
       "1149  ff9ac61f-8f98-4516-9342-7aa1177514d9          24   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2           2   \n",
       "\n",
       "                                                MESSAGE  sentence_id  \\\n",
       "1240                                        i need help            0   \n",
       "1883  i am getting a phone for my son and looking to...            0   \n",
       "859                                         data + talk            0   \n",
       "993                                     less than 10 gb            1   \n",
       "1284                                              10 gb            2   \n",
       "883                                         data + talk            3   \n",
       "1062                                    more than 10 gb            4   \n",
       "1149                                 15 gb freedom data            5   \n",
       "3049   i purchased the cannes outdoor sofa however i...            1   \n",
       "3049                            are they macne washable            2   \n",
       "\n",
       "                                  session_id  word_count  \\\n",
       "1240  ff097a90-5972-46d1-845a-3c9a399b70bc_0           3   \n",
       "1883  ff93a689-8d1a-49c2-a5bc-54dd321707a3_0          26   \n",
       "859   ff9ac61f-8f98-4516-9342-7aa1177514d9_0           3   \n",
       "993   ff9ac61f-8f98-4516-9342-7aa1177514d9_1           4   \n",
       "1284  ff9ac61f-8f98-4516-9342-7aa1177514d9_2           2   \n",
       "883   ff9ac61f-8f98-4516-9342-7aa1177514d9_3           3   \n",
       "1062  ff9ac61f-8f98-4516-9342-7aa1177514d9_4           4   \n",
       "1149  ff9ac61f-8f98-4516-9342-7aa1177514d9_5           4   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_1          17   \n",
       "3049  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2_2           5   \n",
       "\n",
       "                                         processed_text  \n",
       "1240                                       [need, help]  \n",
       "1883   [get, phone, son, look, plan, iphon, se, packag]  \n",
       "859                                        [data, talk]  \n",
       "993                                                [gb]  \n",
       "1284                                               [gb]  \n",
       "883                                        [data, talk]  \n",
       "1062                                               [gb]  \n",
       "1149                                [gb, freedom, data]  \n",
       "3049  [purchas, cann, outdoor, sofa, care, instruct,...  \n",
       "3049                                    [macn, washabl]  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# preprocess text\n",
    "df[\"MESSAGE\"] = df[\"MESSAGE\"].str.replace(\"|\".join(remove_words),'')# remove certain texts from messages\n",
    "df['word_count'] = df['MESSAGE'].apply(word_count)\n",
    "df['processed_text'] = df['MESSAGE'].apply(process_text)\n",
    "df = df[df['processed_text'].map(lambda d: len(d)) > 0]\n",
    "print(df.shape)\n",
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2dd89af6-3abc-4869-b927-f9758e4c00f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[df[\"conversation_id\"]==\"1895298307224761\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8390d6b4-0594-4465-b6ec-e004207c27ba",
   "metadata": {},
   "source": [
    "## split df into 2 dfs\n",
    "- df containing only 1 word sentences\n",
    "- df containing multiple sentences\n",
    "\n",
    "**ignore df containing only 1 word sentences for now**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f5e3a1bd-3309-42b6-b152-ef855a974898",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2608</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>i need to activate new sim</td>\n",
       "      <td>1</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_1</td>\n",
       "      <td>7</td>\n",
       "      <td>[need, activ, new, sim]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2608</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>but it's got getting activated online</td>\n",
       "      <td>2</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_2</td>\n",
       "      <td>7</td>\n",
       "      <td>[got, get, activ, onlin]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987</th>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df</td>\n",
       "      <td>5</td>\n",
       "      <td>account details</td>\n",
       "      <td>0</td>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df_0</td>\n",
       "      <td>2</td>\n",
       "      <td>[account, detail]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3463</th>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504</td>\n",
       "      <td>1</td>\n",
       "      <td>i would like to cancel service</td>\n",
       "      <td>0</td>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504_0</td>\n",
       "      <td>6</td>\n",
       "      <td>[cancel, servic]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2591</th>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2</td>\n",
       "      <td>5</td>\n",
       "      <td>i have a balance of  but my wife 's account ag...</td>\n",
       "      <td>0</td>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2_0</td>\n",
       "      <td>16</td>\n",
       "      <td>[balanc, wife, account, say, suspend]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           conversation_id  message_id  \\\n",
       "2608  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "2608  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "987   0024cce3-b115-4a79-88c7-604d046076df           5   \n",
       "3463  0028d3f2-964b-4903-bd37-3bb364459504           1   \n",
       "2591  00560815-b4b9-4919-bd75-5dbd07aadec2           5   \n",
       "\n",
       "                                                MESSAGE  sentence_id  \\\n",
       "2608                         i need to activate new sim            1   \n",
       "2608              but it's got getting activated online            2   \n",
       "987                                     account details            0   \n",
       "3463                     i would like to cancel service            0   \n",
       "2591  i have a balance of  but my wife 's account ag...            0   \n",
       "\n",
       "                                  session_id  word_count  \\\n",
       "2608  001bede7-810d-4284-b289-c29ab8a12bba_1           7   \n",
       "2608  001bede7-810d-4284-b289-c29ab8a12bba_2           7   \n",
       "987   0024cce3-b115-4a79-88c7-604d046076df_0           2   \n",
       "3463  0028d3f2-964b-4903-bd37-3bb364459504_0           6   \n",
       "2591  00560815-b4b9-4919-bd75-5dbd07aadec2_0          16   \n",
       "\n",
       "                             processed_text  \n",
       "2608                [need, activ, new, sim]  \n",
       "2608               [got, get, activ, onlin]  \n",
       "987                       [account, detail]  \n",
       "3463                       [cancel, servic]  \n",
       "2591  [balanc, wife, account, say, suspend]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = df[df[\"processed_text\"].str.len()==1]\n",
    "df2 = df[df[\"processed_text\"].str.len()!=1]\n",
    "df2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "34046163-b3f2-4ed5-a34f-d9ad1753bcfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.drop_duplicates(subset=[\"MESSAGE\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b528f86c-2c00-4491-b730-bc599379e652",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2495.000000\n",
       "mean       10.488577\n",
       "std         7.580401\n",
       "min         1.000000\n",
       "25%         6.000000\n",
       "50%         9.000000\n",
       "75%        13.000000\n",
       "max       144.000000\n",
       "Name: word_count, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2['word_count'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "179611a3-a3db-40e5-bb22-d066e951687a",
   "metadata": {},
   "source": [
    "# NMF on multi-word sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0f19833c-11a4-44b8-a3f7-701adf9b466b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1372"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_text = [item for sublist in df2.processed_text for item in sublist]\n",
    "len(set(processed_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "724321db-6bbe-44b0-8473-a4a8c4377354",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use Gensim's NMF to get the best num of topics via coherence score\n",
    "texts = df2['processed_text']\n",
    "\n",
    "# Create a dictionary\n",
    "# In gensim a dictionary is a mapping between words and their integer id\n",
    "dictionary = Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4afde9ab-67e5-4ed7-b55d-b2218f7ab311",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Create a list of the topic numbers we want to try\n",
    "# topic_nums = list(np.arange(100, 200 + 1, 5))\n",
    "\n",
    "# # Run the nmf model and calculate the coherence score\n",
    "# # for each number of topics\n",
    "# coherence_scores = []\n",
    "\n",
    "# for num in tqdm(topic_nums):\n",
    "#     nmf = Nmf(\n",
    "#         corpus=corpus,\n",
    "#         num_topics=num,\n",
    "#         id2word=dictionary,\n",
    "#         chunksize=2000,\n",
    "#         passes=5,\n",
    "#         kappa=.1,\n",
    "#         minimum_probability=0.01,\n",
    "#         w_max_iter=300,\n",
    "#         w_stop_condition=0.0001,\n",
    "#         h_max_iter=100,\n",
    "#         h_stop_condition=0.001,\n",
    "#         eval_every=10,\n",
    "#         normalize=True,\n",
    "#         random_state=42\n",
    "#     )\n",
    "    \n",
    "#     # Run the coherence model to get the score\n",
    "#     cm = CoherenceModel(\n",
    "#         model=nmf,\n",
    "#         texts=texts,\n",
    "#         dictionary=dictionary,\n",
    "#         coherence='c_v'\n",
    "#     )\n",
    "    \n",
    "#     coherence_scores.append(round(cm.get_coherence(), 5))\n",
    "\n",
    "# # Get the number of topics with the highest coherence score\n",
    "# scores = list(zip(topic_nums, coherence_scores))\n",
    "# best_num_topics = sorted(scores, key=itemgetter(1), reverse=True)[0][0]\n",
    "\n",
    "# # Plot the results\n",
    "# fig = plt.figure(figsize=(30, 12))\n",
    "\n",
    "# plt.plot(\n",
    "#     topic_nums,\n",
    "#     coherence_scores,\n",
    "#     linewidth=3,\n",
    "#     color='#4287f5'\n",
    "# )\n",
    "\n",
    "# plt.xlabel(\"Topic Num\", fontsize=14)\n",
    "# plt.ylabel(\"Coherence Score\", fontsize=14)\n",
    "# plt.title('Coherence Score by Topic Number - Best Number of Topics: {}'.format(best_num_topics), fontsize=18)\n",
    "# plt.xticks(np.arange(5, max(topic_nums) + 1, 5), fontsize=12)\n",
    "# plt.yticks(fontsize=12)\n",
    "\n",
    "# file_name = 'c_score'\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e4f511c7-51c1-4349-bdff-7032e4028603",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now use the number of topics with the \n",
    "# highest coherence score to run the \n",
    "# sklearn nmf model\n",
    "\n",
    "texts = df2['processed_text']\n",
    "\n",
    "# Create the tfidf weights\n",
    "tfidf_vectorizer = TfidfVectorizer(\n",
    "    min_df=3,\n",
    "    max_df=0.9,\n",
    "    max_features=10000,\n",
    "    ngram_range=(2, 5),\n",
    "    preprocessor=' '.join\n",
    ")\n",
    "\n",
    "tfidf = tfidf_vectorizer.fit_transform(texts)\n",
    "\n",
    "# Save the feature names for later to create topic summaries\n",
    "tfidf_fn = tfidf_vectorizer.get_feature_names()\n",
    "\n",
    "# Run the nmf model\n",
    "nmf = NMF(\n",
    "    n_components=best_num_topics,\n",
    "    init='nndsvd',\n",
    "    max_iter=1000,\n",
    "    l1_ratio=0.0,\n",
    "    solver='cd',\n",
    "    alpha=0.0,\n",
    "    tol=1e-4,\n",
    "    random_state=42\n",
    ").fit(tfidf)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9791a8f0",
   "metadata": {},
   "source": [
    "## Generate topic keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e913f281-3b61-45cc-89d9-aaddff477cc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['phone number',\n",
       " 'number neacct',\n",
       " 'phone number account',\n",
       " 'suspend phone',\n",
       " 'suspend phone number']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a dict mapping of topic name and top k keywords\n",
    "docweights = nmf.transform(tfidf_vectorizer.transform(texts))\n",
    "topic_dict = get_topic_dictionary(\n",
    "    nmf,\n",
    "    tfidf_fn,\n",
    "    n_top_words\n",
    ")\n",
    "topic_dict[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40bcbfa1",
   "metadata": {},
   "source": [
    "### Get top k topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4dabc7bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([101,  28,  12]),\n",
       " array([-1, -1, -1]),\n",
       " array([-1, -1, -1]),\n",
       " array([21, 67, 68]))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# can go along each row\n",
    "\n",
    "\n",
    "top_k_topics = get_top_k_topics(docweights, top_k = top_k)\n",
    "top_k_topics[0], top_k_topics[1], top_k_topics[2], top_k_topics[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd9474c1",
   "metadata": {},
   "source": [
    "### map topics to their keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "57822a5d-5ffd-4984-ba27-c5ee0a87de9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_1_key_words</th>\n",
       "      <th>topic_2</th>\n",
       "      <th>topic_2_key_words</th>\n",
       "      <th>topic_3</th>\n",
       "      <th>topic_3_key_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_1</td>\n",
       "      <td>101</td>\n",
       "      <td>[activ new, activ new sim, need activ, post pa...</td>\n",
       "      <td>28</td>\n",
       "      <td>[new sim, new sim card, need new sim, need new...</td>\n",
       "      <td>12</td>\n",
       "      <td>[activ sim, activ sim card, want activ, sim ph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_2</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df_0</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504_0</td>\n",
       "      <td>21</td>\n",
       "      <td>[cancel servic, servic freedom, freedom mobil ...</td>\n",
       "      <td>67</td>\n",
       "      <td>[upgrad devic, doe work, hey want, good night,...</td>\n",
       "      <td>68</td>\n",
       "      <td>[data work, year contract, hey want, good nigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2_0</td>\n",
       "      <td>136</td>\n",
       "      <td>[postal code, account say, payment account, sa...</td>\n",
       "      <td>105</td>\n",
       "      <td>[check account, charg time, payment account, w...</td>\n",
       "      <td>99</td>\n",
       "      <td>[log account, account set, account say, say el...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id  topic_1  \\\n",
       "0  001bede7-810d-4284-b289-c29ab8a12bba_1      101   \n",
       "1  001bede7-810d-4284-b289-c29ab8a12bba_2       -1   \n",
       "2  0024cce3-b115-4a79-88c7-604d046076df_0       -1   \n",
       "3  0028d3f2-964b-4903-bd37-3bb364459504_0       21   \n",
       "4  00560815-b4b9-4919-bd75-5dbd07aadec2_0      136   \n",
       "\n",
       "                                   topic_1_key_words  topic_2  \\\n",
       "0  [activ new, activ new sim, need activ, post pa...       28   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [cancel servic, servic freedom, freedom mobil ...       67   \n",
       "4  [postal code, account say, payment account, sa...      105   \n",
       "\n",
       "                                   topic_2_key_words  topic_3  \\\n",
       "0  [new sim, new sim card, need new sim, need new...       12   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [upgrad devic, doe work, hey want, good night,...       68   \n",
       "4  [check account, charg time, payment account, w...       99   \n",
       "\n",
       "                                   topic_3_key_words  \n",
       "0  [activ sim, activ sim card, want activ, sim ph...  \n",
       "1                                                     \n",
       "2                                                     \n",
       "3  [data work, year contract, hey want, good nigh...  \n",
       "4  [log account, account set, account say, say el...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a temp df with the sesion id and topic number to join on\n",
    "# with top k topics\n",
    "session_ids = df2['session_id'].tolist()\n",
    "\n",
    "df_merge = pd.DataFrame({\n",
    "    'session_id': session_ids,\n",
    "})\n",
    "\n",
    "# create 2*k columns, each wtih top k topics and their associated keywords\n",
    "for i in range(top_k_topics.shape[1]):\n",
    "    df_merge[f\"topic_{i+1}\"] = top_k_topics[:,i]\n",
    "    df_merge[f\"topic_{i+1}_key_words\"] = df_merge[f\"topic_{i+1}\"].map(topic_dict)\n",
    "# df_merge[\"topic_key_words\"] = df_merge[\"topic\"].map(topic_dict)\n",
    "df_merge.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d04bf6da-1a11-4a2e-a679-5d6dc5e52f58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_1_key_words</th>\n",
       "      <th>topic_2</th>\n",
       "      <th>topic_2_key_words</th>\n",
       "      <th>topic_3</th>\n",
       "      <th>topic_3_key_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>i need to activate new sim</td>\n",
       "      <td>1</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_1</td>\n",
       "      <td>7</td>\n",
       "      <td>[need, activ, new, sim]</td>\n",
       "      <td>101</td>\n",
       "      <td>[activ new, activ new sim, need activ, post pa...</td>\n",
       "      <td>28</td>\n",
       "      <td>[new sim, new sim card, need new sim, need new...</td>\n",
       "      <td>12</td>\n",
       "      <td>[activ sim, activ sim card, want activ, sim ph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>but it's got getting activated online</td>\n",
       "      <td>2</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_2</td>\n",
       "      <td>7</td>\n",
       "      <td>[got, get, activ, onlin]</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df</td>\n",
       "      <td>5</td>\n",
       "      <td>account details</td>\n",
       "      <td>0</td>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df_0</td>\n",
       "      <td>2</td>\n",
       "      <td>[account, detail]</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504</td>\n",
       "      <td>1</td>\n",
       "      <td>i would like to cancel service</td>\n",
       "      <td>0</td>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504_0</td>\n",
       "      <td>6</td>\n",
       "      <td>[cancel, servic]</td>\n",
       "      <td>21</td>\n",
       "      <td>[cancel servic, servic freedom, freedom mobil ...</td>\n",
       "      <td>67</td>\n",
       "      <td>[upgrad devic, doe work, hey want, good night,...</td>\n",
       "      <td>68</td>\n",
       "      <td>[data work, year contract, hey want, good nigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2</td>\n",
       "      <td>5</td>\n",
       "      <td>i have a balance of  but my wife 's account ag...</td>\n",
       "      <td>0</td>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2_0</td>\n",
       "      <td>16</td>\n",
       "      <td>[balanc, wife, account, say, suspend]</td>\n",
       "      <td>136</td>\n",
       "      <td>[postal code, account say, payment account, sa...</td>\n",
       "      <td>105</td>\n",
       "      <td>[check account, charg time, payment account, w...</td>\n",
       "      <td>99</td>\n",
       "      <td>[log account, account set, account say, say el...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        conversation_id  message_id  \\\n",
       "0  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "1  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "2  0024cce3-b115-4a79-88c7-604d046076df           5   \n",
       "3  0028d3f2-964b-4903-bd37-3bb364459504           1   \n",
       "4  00560815-b4b9-4919-bd75-5dbd07aadec2           5   \n",
       "\n",
       "                                             MESSAGE  sentence_id  \\\n",
       "0                         i need to activate new sim            1   \n",
       "1              but it's got getting activated online            2   \n",
       "2                                    account details            0   \n",
       "3                     i would like to cancel service            0   \n",
       "4  i have a balance of  but my wife 's account ag...            0   \n",
       "\n",
       "                               session_id  word_count  \\\n",
       "0  001bede7-810d-4284-b289-c29ab8a12bba_1           7   \n",
       "1  001bede7-810d-4284-b289-c29ab8a12bba_2           7   \n",
       "2  0024cce3-b115-4a79-88c7-604d046076df_0           2   \n",
       "3  0028d3f2-964b-4903-bd37-3bb364459504_0           6   \n",
       "4  00560815-b4b9-4919-bd75-5dbd07aadec2_0          16   \n",
       "\n",
       "                          processed_text  topic_1  \\\n",
       "0                [need, activ, new, sim]      101   \n",
       "1               [got, get, activ, onlin]       -1   \n",
       "2                      [account, detail]       -1   \n",
       "3                       [cancel, servic]       21   \n",
       "4  [balanc, wife, account, say, suspend]      136   \n",
       "\n",
       "                                   topic_1_key_words  topic_2  \\\n",
       "0  [activ new, activ new sim, need activ, post pa...       28   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [cancel servic, servic freedom, freedom mobil ...       67   \n",
       "4  [postal code, account say, payment account, sa...      105   \n",
       "\n",
       "                                   topic_2_key_words  topic_3  \\\n",
       "0  [new sim, new sim card, need new sim, need new...       12   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [upgrad devic, doe work, hey want, good night,...       68   \n",
       "4  [check account, charg time, payment account, w...       99   \n",
       "\n",
       "                                   topic_3_key_words  \n",
       "0  [activ sim, activ sim card, want activ, sim ph...  \n",
       "1                                                     \n",
       "2                                                     \n",
       "3  [data work, year contract, hey want, good nigh...  \n",
       "4  [log account, account set, account say, say el...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merging to get the topic and key words\n",
    "df_topics2 = pd.merge(\n",
    "    df2,\n",
    "    df_merge,\n",
    "    on='session_id',\n",
    "    how='left'\n",
    ")\n",
    "df_topics2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3a840a1",
   "metadata": {},
   "source": [
    "### Topics that pass a certain threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a9faad0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 0.7908036657609471)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.amin(docweights), np.amax(docweights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a8efb026-9c48-4597-9b41-571aead83c47",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "df_topics2[f\"topics_threshold_>={docweights_threshold}\"] = threshold_docweights(docweights, threshold = docweights_threshold)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a4a482f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>message_id</th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_1_key_words</th>\n",
       "      <th>topic_2</th>\n",
       "      <th>topic_2_key_words</th>\n",
       "      <th>topic_3</th>\n",
       "      <th>topic_3_key_words</th>\n",
       "      <th>topics_threshold_&gt;=0.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>i need to activate new sim</td>\n",
       "      <td>1</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_1</td>\n",
       "      <td>7</td>\n",
       "      <td>[need, activ, new, sim]</td>\n",
       "      <td>101</td>\n",
       "      <td>[activ new, activ new sim, need activ, post pa...</td>\n",
       "      <td>28</td>\n",
       "      <td>[new sim, new sim card, need new sim, need new...</td>\n",
       "      <td>12</td>\n",
       "      <td>[activ sim, activ sim card, want activ, sim ph...</td>\n",
       "      <td>[28, 101]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba</td>\n",
       "      <td>2</td>\n",
       "      <td>but it's got getting activated online</td>\n",
       "      <td>2</td>\n",
       "      <td>001bede7-810d-4284-b289-c29ab8a12bba_2</td>\n",
       "      <td>7</td>\n",
       "      <td>[got, get, activ, onlin]</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df</td>\n",
       "      <td>5</td>\n",
       "      <td>account details</td>\n",
       "      <td>0</td>\n",
       "      <td>0024cce3-b115-4a79-88c7-604d046076df_0</td>\n",
       "      <td>2</td>\n",
       "      <td>[account, detail]</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504</td>\n",
       "      <td>1</td>\n",
       "      <td>i would like to cancel service</td>\n",
       "      <td>0</td>\n",
       "      <td>0028d3f2-964b-4903-bd37-3bb364459504_0</td>\n",
       "      <td>6</td>\n",
       "      <td>[cancel, servic]</td>\n",
       "      <td>21</td>\n",
       "      <td>[cancel servic, servic freedom, freedom mobil ...</td>\n",
       "      <td>67</td>\n",
       "      <td>[upgrad devic, doe work, hey want, good night,...</td>\n",
       "      <td>68</td>\n",
       "      <td>[data work, year contract, hey want, good nigh...</td>\n",
       "      <td>[21]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2</td>\n",
       "      <td>5</td>\n",
       "      <td>i have a balance of  but my wife 's account ag...</td>\n",
       "      <td>0</td>\n",
       "      <td>00560815-b4b9-4919-bd75-5dbd07aadec2_0</td>\n",
       "      <td>16</td>\n",
       "      <td>[balanc, wife, account, say, suspend]</td>\n",
       "      <td>136</td>\n",
       "      <td>[postal code, account say, payment account, sa...</td>\n",
       "      <td>105</td>\n",
       "      <td>[check account, charg time, payment account, w...</td>\n",
       "      <td>99</td>\n",
       "      <td>[log account, account set, account say, say el...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        conversation_id  message_id  \\\n",
       "0  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "1  001bede7-810d-4284-b289-c29ab8a12bba           2   \n",
       "2  0024cce3-b115-4a79-88c7-604d046076df           5   \n",
       "3  0028d3f2-964b-4903-bd37-3bb364459504           1   \n",
       "4  00560815-b4b9-4919-bd75-5dbd07aadec2           5   \n",
       "\n",
       "                                             MESSAGE  sentence_id  \\\n",
       "0                         i need to activate new sim            1   \n",
       "1              but it's got getting activated online            2   \n",
       "2                                    account details            0   \n",
       "3                     i would like to cancel service            0   \n",
       "4  i have a balance of  but my wife 's account ag...            0   \n",
       "\n",
       "                               session_id  word_count  \\\n",
       "0  001bede7-810d-4284-b289-c29ab8a12bba_1           7   \n",
       "1  001bede7-810d-4284-b289-c29ab8a12bba_2           7   \n",
       "2  0024cce3-b115-4a79-88c7-604d046076df_0           2   \n",
       "3  0028d3f2-964b-4903-bd37-3bb364459504_0           6   \n",
       "4  00560815-b4b9-4919-bd75-5dbd07aadec2_0          16   \n",
       "\n",
       "                          processed_text  topic_1  \\\n",
       "0                [need, activ, new, sim]      101   \n",
       "1               [got, get, activ, onlin]       -1   \n",
       "2                      [account, detail]       -1   \n",
       "3                       [cancel, servic]       21   \n",
       "4  [balanc, wife, account, say, suspend]      136   \n",
       "\n",
       "                                   topic_1_key_words  topic_2  \\\n",
       "0  [activ new, activ new sim, need activ, post pa...       28   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [cancel servic, servic freedom, freedom mobil ...       67   \n",
       "4  [postal code, account say, payment account, sa...      105   \n",
       "\n",
       "                                   topic_2_key_words  topic_3  \\\n",
       "0  [new sim, new sim card, need new sim, need new...       12   \n",
       "1                                                          -1   \n",
       "2                                                          -1   \n",
       "3  [upgrad devic, doe work, hey want, good night,...       68   \n",
       "4  [check account, charg time, payment account, w...       99   \n",
       "\n",
       "                                   topic_3_key_words topics_threshold_>=0.1  \n",
       "0  [activ sim, activ sim card, want activ, sim ph...              [28, 101]  \n",
       "1                                                                        []  \n",
       "2                                                                        []  \n",
       "3  [data work, year contract, hey want, good nigh...                   [21]  \n",
       "4  [log account, account set, account say, say el...                     []  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_topics2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1d9c4b1-fadc-47b2-ac94-c71209ccdd43",
   "metadata": {},
   "source": [
    "# NMF on multi-word sentences\n",
    "\n",
    "Currently not doing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "353482cb-6930-4af7-a291-9f365ff45137",
   "metadata": {},
   "source": [
    "# Aggregate at conversation level text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "953ff186-72e7-48d0-8545-978aa5cd43ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Combine df_topics_1 and df_topics_2\n",
    "\n",
    "df_topics = df_topics2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6a7ea7c0-9f71-492b-924c-b206a1c69839",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort by conversation and sentence id to ensure order\n",
    "df_conv_topics = df_topics.sort_values(by=[\"conversation_id\", \"sentence_id\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5ca6846e-fb4a-472a-9e68-7310431f44e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check \n",
    "# df_conv_topics[df_conv_topics[\"conversation_id\"]==\"1895298307224761\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f0ec866c-3f0b-4283-8ac1-a63b86002d97",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_conv_topics[\"concat_text\"] = df_conv_topics.groupby([\"conversation_id\"])[\"MESSAGE\"].transform(\n",
    "                lambda x: \" \".join(x)\n",
    "            )\n",
    "\n",
    "# generate new dfs\n",
    "df_conv_concat_text = df_conv_topics.groupby([\"conversation_id\"])[\"processed_text\"].agg(sum).reset_index(name=\"preprocessed_concat_text\")\n",
    "df_combined_topics = df_conv_topics.groupby([\"conversation_id\"])[\"topic_1\"].apply(list).reset_index(name=\"combined_topics\")\n",
    "# df_combined_key_words = df_conv_topics.groupby([\"conversation_id\"])[\"topic_key_words\"].apply(list).reset_index(name=\"combined_topic_key_words\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f504e1f6-e48d-4627-b4b5-d7115dcdb14a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# join dfs\n",
    "df_conv_topics = df_conv_topics.merge(df_conv_concat_text, on=\"conversation_id\")\\\n",
    "                    .merge(df_combined_topics, on=\"conversation_id\")\\\n",
    "                    .drop_duplicates(subset=[\"conversation_id\"])\\\n",
    "                    .reset_index()\\\n",
    "                    [[\"conversation_id\", \"concat_text\", \"preprocessed_concat_text\", \"combined_topics\"]]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d11d4361-8644-4d35-b79d-9ea46cab61a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>concat_text</th>\n",
       "      <th>preprocessed_concat_text</th>\n",
       "      <th>combined_topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1204</th>\n",
       "      <td>fe64aeec-c81a-4759-8ac1-8e6257293a12</td>\n",
       "      <td>how does the 99/yr prepaid plan work in terms...</td>\n",
       "      <td>[doe, yr, prepaid, plan, work, term, roam, cha...</td>\n",
       "      <td>[78, -1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1205</th>\n",
       "      <td>fe8a47dd-d103-44a9-9d3f-d88400aa31d8</td>\n",
       "      <td>i need to cancel my mother's phone plan  she ...</td>\n",
       "      <td>[need, cancel, mother, phone, plan, alzheim, r...</td>\n",
       "      <td>[15, 148, 126, -1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1206</th>\n",
       "      <td>fed5e9f2-ff88-453d-9461-c98f5a97af9d</td>\n",
       "      <td>billing information# my bill story# i 'm looki...</td>\n",
       "      <td>[bill, inform, bill, stori, look, statement, f...</td>\n",
       "      <td>[-1, 53, -1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1207</th>\n",
       "      <td>ff93a689-8d1a-49c2-a5bc-54dd321707a3</td>\n",
       "      <td>i am getting a phone for my son and looking to...</td>\n",
       "      <td>[get, phone, son, look, plan, iphon, se, packag]</td>\n",
       "      <td>[110]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1208</th>\n",
       "      <td>ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2</td>\n",
       "      <td>i purchased the cannes outdoor sofa however i...</td>\n",
       "      <td>[purchas, cann, outdoor, sofa, care, instruct,...</td>\n",
       "      <td>[-1, -1]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           conversation_id  \\\n",
       "1204  fe64aeec-c81a-4759-8ac1-8e6257293a12   \n",
       "1205  fe8a47dd-d103-44a9-9d3f-d88400aa31d8   \n",
       "1206  fed5e9f2-ff88-453d-9461-c98f5a97af9d   \n",
       "1207  ff93a689-8d1a-49c2-a5bc-54dd321707a3   \n",
       "1208  ffbf37b3-2b93-4875-9cbb-d5f5dc0ed3e2   \n",
       "\n",
       "                                            concat_text  \\\n",
       "1204   how does the 99/yr prepaid plan work in terms...   \n",
       "1205   i need to cancel my mother's phone plan  she ...   \n",
       "1206  billing information# my bill story# i 'm looki...   \n",
       "1207  i am getting a phone for my son and looking to...   \n",
       "1208   i purchased the cannes outdoor sofa however i...   \n",
       "\n",
       "                               preprocessed_concat_text     combined_topics  \n",
       "1204  [doe, yr, prepaid, plan, work, term, roam, cha...            [78, -1]  \n",
       "1205  [need, cancel, mother, phone, plan, alzheim, r...  [15, 148, 126, -1]  \n",
       "1206  [bill, inform, bill, stori, look, statement, f...     [-1, 53, -1, 0]  \n",
       "1207   [get, phone, son, look, plan, iphon, se, packag]               [110]  \n",
       "1208  [purchas, cann, outdoor, sofa, care, instruct,...            [-1, -1]  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_conv_topics.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8f8bdaa-0581-46f4-a916-be5ce09e3c43",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check \n",
    "# df_conv_topics[df_conv_topics[\"conversation_id\"]==\"1895298307224761\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32aeffac-5bc2-4901-a317-9cc2d142b9ab",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d2dac8aa-b375-48f7-b4dc-93f67dccf782",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results, vectorizer, and nmf model\n",
    "df_topics.to_csv(f\"nmf_topics/{save_folder}/freedom_msg_level_nmf_clusters.csv\", index=False)\n",
    "df_conv_topics.to_csv(f\"nmf_topics/{save_folder}/freedom_conv_level_nmf_clusters.csv\", index=False)\n",
    "dump(tfidf_vectorizer, f\"nmf_models/{save_folder}/msg_level_vectorizer.joblib\")\n",
    "dump(nmf, f\"nmf_models/{save_folder}/msg_level_nmf.joblib\")\n",
    "with open(f\"nmf_topics/{save_folder}/topic_mapping.json\", 'w') as f:\n",
    "    json.dump(topic_dict, f, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50c74a0d",
   "metadata": {},
   "source": [
    "# Get sub-topics from topic 7\n",
    "\n",
    "need to reprocess `processed_text` with `df2['processed_text'] = df2['MESSAGE'].apply(process_text)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00efff96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}