system_specific_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook runs the system specific experiments in \"A Robust and Efficient Stopping Criteria for Systematic Reviews Using Poisson Processes.\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORT LIBRARIES\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from scipy.stats import poisson\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.optimize import curve_fit\n",
    "import operator\n",
    "import random\n",
    "\n",
    "\n",
    "# IMPORT EXPERIEMENTAL FUNCTIONS\n",
    "from utils.read_data_fns import *\n",
    "from utils.target_method_fns import *  \n",
    "from utils.knee_method_fns import *   \n",
    "from utils.inhomogeneous_pp_fns import *  \n",
    "from utils.eval_fns import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N. train topics: 10 N. test topics: 20\n",
      "Total test docs: 78019\n"
     ]
    }
   ],
   "source": [
    "# SET UP\n",
    "\n",
    "# READ TOPIC RELEVANCE DATA\n",
    "with open('data/relevance/qrel_abs_test.txt', 'r') as infile:\n",
    "    qrels_data = infile.readlines()\n",
    "\n",
    "    \n",
    "# CHOOSE SPECIFIC RUN\n",
    "with open('data/runs2017_table3/Waterloo/B-rank-normal.txt', 'r') as infile:\n",
    "    run_data = infile.readlines()\n",
    "\n",
    "    \n",
    "# MAKE RANK AND RELEVANCE DICTIONARIES\n",
    "doc_rank_dic = make_rank_dic(run_data)  # make dictionary of ranked docids for each queryid\n",
    "query_rel_dic = make_rel_dic(qrels_data) # make dictionary of list of docids relevant to each queryid\n",
    "rank_rel_dic = make_rank_rel_dic(query_rel_dic,doc_rank_dic) # make dic of list relevances of ranked docs for each queryid\n",
    "\n",
    "\n",
    "# RANDONLY SPLIT TOPICS INTO TRAIN AND TEST SETS\n",
    "topics_list = make_topics_list(doc_rank_dic,1)  # sort topics by no docs\n",
    "random.seed(1)\n",
    "random.shuffle(topics_list)\n",
    "topics_train = topics_list[0:10]\n",
    "topics_test = topics_list[10:]\n",
    "print(\"N. train topics:\",len(topics_train), \"N. test topics:\",len(topics_test))\n",
    "test_docs_total = np.sum([len(doc_rank_dic[query_id])for query_id in topics_test])\n",
    "print(\"Total test docs:\", test_docs_total)\n",
    "\n",
    "\n",
    "# SET FIXED PARAMETERS \n",
    "n_windows = 10  # number of windows to male from sample for PP\n",
    "des_recalls = [0.5,0.6,0.7, 0.8, 0.9, 0.95]\n",
    "knee_target_ratio = 6 # knee method rho (Cormack and Grossman set to 6)\n",
    "\n",
    "\n",
    "# PP PARAMETERS TO BE TUNED\n",
    "sample_props_list = [[0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,\n",
    "                0.7,0.75,0.8,0.85,0.9,0.95,1], \n",
    "                    [0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,\n",
    "                0.7,0.75,0.8,0.85,0.9,0.95,1],\n",
    "                     [0.6,0.65,\n",
    "                0.7,0.75,0.8,0.85,0.9,0.95,1]  \n",
    "                    ]  # proportion of docs to sample\n",
    "min_rel_in_sample_list = [5,20,50] # min number rel docs must be initial sample to proceed with algorithm \n",
    "pp_adjusts = []\n",
    "for sample_props in sample_props_list:\n",
    "    for min_rel_in_sample in min_rel_in_sample_list:\n",
    "        #adjust = \"ms:\"+str(sample_props[0])+\" mr:\"+str(min_rel_in_sample)\n",
    "        adjust = [sample_props,min_rel_in_sample]\n",
    "        pp_adjusts.append(adjust)\n",
    "\n",
    "\n",
    "        \n",
    "# KM PARAMETERS TO BE TUNED\n",
    "km_adjusts = [0, 50, 100, 150, 200]#, 50, 100, 150, 200] # adjustments to target ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KNEE METHOD TUNING FUNCTIONS\n",
    "\n",
    "# FN TO LOOP OVER RUNS, IMPLEMENTING KNEE METHOD FOR EACH ADJUSTMENT VALUE\n",
    "def do_knee_method(topics, adjusts, des_recall):  # des recall only used to name dictionary key\n",
    " \n",
    "    # LOOP OVER QUERIES\n",
    "    score_dic = {}\n",
    "\n",
    "    for query_id in topics:\n",
    "        score_dic[query_id] = []\n",
    "\n",
    "        # EXTRACT COUNTS AND REL LISTS\n",
    "        n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "        rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs   \n",
    "\n",
    "\n",
    "        # KNEE METHOD\n",
    "        batches = get_batches(n_docs)\n",
    "\n",
    "        for adjust in adjusts:\n",
    "            knee, knee_stop = get_knee_stopping_point_var_adjust(rel_list, batches, knee_target_ratio, adjust)[0:2]\n",
    "            knee_recall = calc_recall(rel_list, knee_stop)\n",
    "            knee_effort = knee_stop\n",
    "            knee_accept = calc_accept(knee_recall, des_recall)\n",
    "            score_dic[query_id].append((knee_recall, knee_effort, knee_accept))\n",
    "\n",
    "    rel_vec_dict = {}\n",
    "    eff_vec_dict = {}\n",
    "        \n",
    "    for i, adjust in enumerate(adjusts):    \n",
    "        rel_vec_dict[str(adjust)+\" rel\"] = [val[i][2] for val in score_dic.values()]\n",
    "        eff_vec_dict[str(adjust)+\" eff\"] = [val[i][1] for val in score_dic.values()]\n",
    "\n",
    "    df_score_dic = {} \n",
    "    for key in rel_vec_dict.keys():\n",
    "         df_score_dic[key] = calc_reliability(rel_vec_dict[key])\n",
    "    for key in eff_vec_dict.keys():    \n",
    "           df_score_dic[key] = np.sum(eff_vec_dict[key])\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(df_score_dic, orient='index',columns = [des_recall])\n",
    "    df = df.T\n",
    "    df = df.round(2)\n",
    "    \n",
    "    return df\n",
    "\n",
    "# FN TO CALCULATE BEST KM ASJUSTMENT VALUE\n",
    "def get_best_knee_adjust(knee_low_rel_adjustments, des_prob, des_recall, df):\n",
    "    \n",
    "    end_scores = df.loc[des_recall].to_dict()\n",
    "\n",
    "    over_effs = {}\n",
    "    under_accepts = {}\n",
    "\n",
    "    for adjust in knee_low_rel_adjustments[::-1]:\n",
    "        if end_scores[str(adjust)+\" rel\"] >= des_prob:\n",
    "            over_effs[end_scores[str(adjust)+\" eff\"]] = adjust\n",
    "        else: \n",
    "            under_accepts[end_scores[str(adjust)+\" rel\"]]  = adjust\n",
    "\n",
    "    if len(over_effs) > 0:\n",
    "        min_eff = min(over_effs.keys())\n",
    "        best_adjust = over_effs[min_eff]\n",
    "\n",
    "    else:\n",
    "        max_accept = max(under_accepts.keys())\n",
    "        best_adjust = under_accepts[max_accept]\n",
    "        \n",
    "    return [best_adjust]\n",
    "\n",
    "\n",
    "# POISSON PROCESS TUNING FUNCTIONS\n",
    "\n",
    "# FN TO LOOP OVER RUNS, IMPLEMENTING PP FOR EACH PARAMETER SET\n",
    "def do_pp_method(topics, adjusts,  des_prob, des_recall):\n",
    " \n",
    "    # LOOP OVER QUERIES\n",
    "    score_dic = {}\n",
    "\n",
    "    for query_id in topics:\n",
    "        score_dic[query_id] = []\n",
    "\n",
    "        # EXTRACT COUNTS AND REL LISTS\n",
    "        n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "        rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs   \n",
    "\n",
    "        # INHOMOGENEOUS POISSON PROCESS\n",
    "        \n",
    "        for adjust in adjusts:  \n",
    "            sample_props = adjust[0]\n",
    "            min_rel_in_sample = adjust[1]\n",
    "        \n",
    "            # check topic meets initial relevance requirement\n",
    "            n_samp_docs = int(round(n_docs*sample_props[0]))\n",
    "            sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "\n",
    "            # if meet size requirement run algorithm; else return n_docs as stopping point\n",
    "            if (np.sum(sample_rel_list) >= min_rel_in_sample):\n",
    "\n",
    "                windows_end_point = 0\n",
    "                pred_stop_n = n_docs\n",
    "                i = 0\n",
    "\n",
    "                while (i < len(sample_props)) and (pred_stop_n > n_samp_docs):\n",
    "                    sample_prop = sample_props[i]\n",
    "\n",
    "                    n_samp_docs = int(round(n_docs*sample_props[i]))\n",
    "                    sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "\n",
    "                    # get points\n",
    "                    windows = make_windows(n_windows, sample_prop, n_docs)\n",
    "                    window_size = windows[0][1]\n",
    "\n",
    "                    x,y = get_points(windows, window_size, sample_rel_list)  # calculate points that will be used to fit curve\n",
    "\n",
    "                    good_curve_fit = 0\n",
    "                    # try to fit curve\n",
    "                    try: \n",
    "                        p0 = [0.1, 0.001]  # initialise curve parameters\n",
    "                        opt, pcov = curve_fit(model_func, x, y, p0)  # fit curve\n",
    "                        good_curve_fit = 1\n",
    "                    except: \n",
    "                        pass\n",
    "                    \n",
    "                    if(good_curve_fit == 1):\n",
    "                        a, k = opt\n",
    "                        y2 = model_func(x, a, k) # get y-values for fitted curve\n",
    "                        #y2 = a*np.exp(-k*x)\n",
    "\n",
    "\n",
    "                        # check distance between \"curves\" are at end sample is suffu\n",
    "                        n_rel_at_end_samp = np.sum(sample_rel_list)\n",
    "                        y3 =  model_func(np.array(range(1,len(sample_rel_list)+1)), a, k)\n",
    "                        pred_by_curve_rel_at_stop = np.sum(y3)\n",
    "                        pred_by_curve_rel_at_stop = int(round(pred_by_curve_rel_at_stop))       \n",
    "                        if n_rel_at_end_samp >= des_recall*pred_by_curve_rel_at_stop:\n",
    "\n",
    "                            # using inhom Poisson process with fitted curve as rate fn, \n",
    "                            # predict total number rel docs in topic (subject to min prob)\n",
    "                            mu = (a/-k)*(math.exp(-k*n_docs)-1)  # integral model_func\n",
    "                            pred_n_rel = predict_n_rel(des_prob, n_docs, mu) # predict max number rel docs (using poisson cdf)\n",
    "                            des_n_rel = des_recall*pred_n_rel\n",
    "                            #pred_stop_n = get_stopping_inhom(des_recall, pred_n_rel, rel_list, n_docs)  # use prev value to pred stopping point\n",
    "                            if des_n_rel <= n_rel_at_end_samp:\n",
    "                                pred_stop_n = n_rel_at_end_samp\n",
    "                                \n",
    "\n",
    "                    i += 1  # increase sample proportion size\n",
    "\n",
    "\n",
    "                # score result \n",
    "                inhom_recall = calc_recall(rel_list, n_samp_docs)\n",
    "                inhom_effort = n_samp_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "                    \n",
    "                    \n",
    "            else: # if not enough docs in topic or not enough rel docs in min sample:\n",
    "\n",
    "                inhom_stop_n = n_docs  # take stopping point as n_docs\n",
    "                inhom_recall = calc_recall(rel_list, inhom_stop_n)\n",
    "                inhom_effort = n_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "\n",
    "\n",
    "    rel_vec_dict = {}\n",
    "    eff_vec_dict = {}\n",
    "\n",
    "        \n",
    "    for i, adjust in enumerate(adjusts):    \n",
    "        rel_vec_dict[str(adjust[0][0])+\" \"+str(adjust[1])+\" rel\"] = [val[i][2] for val in score_dic.values()]\n",
    "        eff_vec_dict[str(adjust[0][0])+\" \"+str(adjust[1])+\" eff\"] = [val[i][1] for val in score_dic.values()]\n",
    "\n",
    " \n",
    "    df_score_dic = {} \n",
    "    for key in rel_vec_dict.keys():\n",
    "         df_score_dic[key] = calc_reliability(rel_vec_dict[key])\n",
    "    for key in eff_vec_dict.keys():    \n",
    "           df_score_dic[key] = np.sum(eff_vec_dict[key])\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(df_score_dic, orient='index', columns = [des_recall])\n",
    "    df = df.T\n",
    "    df = df.round(2)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "# FN TO CALCULATE BEST PP ASJUSTMENT VALUE\n",
    "def get_best_pp_adjust(adjusts, des_prob, des_recall, df):\n",
    "    \n",
    "    end_scores = df.loc[des_recall].to_dict()\n",
    "\n",
    "    over_effs = {}\n",
    "    under_accepts = {}\n",
    "\n",
    "    for adjust in adjusts[::-1]:\n",
    "        if end_scores[str(adjust[0][0])+\" \"+str(adjust[1])+\" rel\"] >= des_prob:\n",
    "            over_effs[end_scores[str(adjust[0][0])+\" \"+str(adjust[1])+\" eff\"]] = adjust\n",
    "        else: \n",
    "            under_accepts[end_scores[str(adjust)+\" rel\"]]  = adjust\n",
    "    \n",
    "    if len(over_effs) > 0:\n",
    "        min_eff = min(over_effs.keys())\n",
    "        best_adjust = over_effs[min_eff]\n",
    "\n",
    "    else:\n",
    "        max_accept = max(under_accepts.keys())\n",
    "        best_adjust = under_accepts[max_accept]\n",
    "        \n",
    "    return [best_adjust]\n",
    "\n",
    "\n",
    "# FUNCTIONS FOR UNTUNED METHODS (OR, BL AND TM)\n",
    "\n",
    "# FN TO LOOP OVER TOPICS, IMPLEMENTING TARGET METHOD\n",
    "def do_target_method(topics, des_recall, des_prob):  # des recall only used to name dictionary key\n",
    " \n",
    "    # LOOP OVER QUERIES\n",
    "    score_dic = {}\n",
    "\n",
    "    for query_id in topics:\n",
    "        score_dic[query_id] = []\n",
    "\n",
    "        # EXTRACT COUNTS AND REL LISTS\n",
    "        n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "        rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs   \n",
    "\n",
    "\n",
    "        # TARGET METHOD\n",
    "        random.seed(1)\n",
    "        target_size = get_target_size(des_recall, des_prob)\n",
    "        target_list, examined_list = make_target_set(rel_list, n_docs, target_size)  # get target sample and list all docs examined\n",
    "        tar_stop_n = get_stopping_target(target_list, n_docs, target_size)  # stopping point\n",
    "        all_examined_idxs = get_all_target_examined_idxs(examined_list, tar_stop_n)  # list of every doc examined during method\n",
    "        tar_recall = calc_recall(rel_list, tar_stop_n)\n",
    "        tar_effort = len(all_examined_idxs) # total effort (inc. sampling)\n",
    "        tar_accept = calc_accept(tar_recall, des_recall)\n",
    "        score_dic[query_id].append((tar_recall, tar_effort, tar_accept))\n",
    "    \n",
    "\n",
    "    rel_vec_dict = {}\n",
    "    eff_vec_dict = {}\n",
    "         \n",
    "    rel_vec_dict[\"TM rel\"] = [val[0][2] for val in score_dic.values()]\n",
    "    eff_vec_dict[\"TM eff\"] = [val[0][1] for val in score_dic.values()]\n",
    "\n",
    "    df_score_dic = {} \n",
    "    for key in rel_vec_dict.keys():\n",
    "         df_score_dic[key] = calc_reliability(rel_vec_dict[key])\n",
    "    for key in eff_vec_dict.keys():    \n",
    "           df_score_dic[key] = np.sum(eff_vec_dict[key])\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(df_score_dic, orient='index',columns = [des_recall])\n",
    "    df = df.T\n",
    "    df = df.round(2)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "# FN TO LOOP OVER TOPICS, IMPLEMENTING BASELINE METHOD\n",
    "def do_baseline_method(topics, des_recall): \n",
    " \n",
    "    # LOOP OVER QUERIES\n",
    "    score_dic = {}\n",
    "\n",
    "    for query_id in topics:\n",
    "        score_dic[query_id] = []\n",
    "\n",
    "        # EXTRACT COUNTS AND REL LISTS\n",
    "        n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "        rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs   \n",
    "\n",
    "\n",
    "        # BL METHOD\n",
    "        bl_stop_n = int(n_docs*des_recall)\n",
    "        bl_recall = calc_recall(rel_list, bl_stop_n)\n",
    "        bl_effort = bl_stop_n # total effort (inc. sampling)\n",
    "        bl_accept = calc_accept(bl_recall, des_recall)\n",
    "        score_dic[query_id].append((bl_recall, bl_effort, bl_accept))\n",
    "    \n",
    "\n",
    "    rel_vec_dict = {}\n",
    "    eff_vec_dict = {}\n",
    "         \n",
    "    rel_vec_dict[\"BL rel\"] = [val[0][2] for val in score_dic.values()]\n",
    "    eff_vec_dict[\"BL eff\"] = [val[0][1] for val in score_dic.values()]\n",
    "\n",
    "    df_score_dic = {} \n",
    "    for key in rel_vec_dict.keys():\n",
    "         df_score_dic[key] = calc_reliability(rel_vec_dict[key])\n",
    "    for key in eff_vec_dict.keys():    \n",
    "           df_score_dic[key] = np.sum(eff_vec_dict[key])\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(df_score_dic, orient='index',columns = [des_recall])\n",
    "    df = df.T\n",
    "    df = df.round(2)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "# FN TO LOOP OVER TOPICS, IMPLEMENTING ORACLE METHOD\n",
    "def do_oracle_method(topics, des_recall):  # des recall only used to name dictionary key\n",
    " \n",
    "    # LOOP OVER QUERIES\n",
    "    score_dic = {}\n",
    "\n",
    "    for query_id in topics:\n",
    "        score_dic[query_id] = []\n",
    "\n",
    "        # EXTRACT COUNTS AND REL LISTS\n",
    "        n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "        rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs   \n",
    "\n",
    "       # ORACLE\n",
    "        rel_doc_idxs = np.where(np.array(rel_list) == 1)[0]\n",
    "        orcale_n_rel = math.ceil(len(rel_doc_idxs)*des_recall)\n",
    "        oracle_idx = rel_doc_idxs[orcale_n_rel-1]\n",
    "        oracle_eff = oracle_idx+1\n",
    "        score_dic[query_id].append(oracle_eff)\n",
    "      \n",
    "\n",
    "    eff_vec_dict = {}\n",
    "         \n",
    "    eff_vec_dict[\"OR eff\"] = [val[0] for val in score_dic.values()]\n",
    "\n",
    "    df_score_dic = {} \n",
    "    for key in eff_vec_dict.keys():    \n",
    "           df_score_dic[key] = np.sum(eff_vec_dict[key])\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(df_score_dic, orient='index',columns = [des_recall])\n",
    "    df = df.T\n",
    "    df = df.round(2)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "# FN TO CALCULATE PERCENTAGE OF EFFORT SAVED\n",
    "def pes(eff): \n",
    "    ts = test_docs_total - eff\n",
    "    ts = round(100*ts/test_docs_total,1)\n",
    "    return ts\n",
    "\n",
    "\n",
    "# FN TO CLEAN RESULTS FOR LATEX TABLE\n",
    "def get_clean_results_dict(df_test, des_prob, name):\n",
    "    sd = {}\n",
    "    \n",
    "    if name == \"OR\":\n",
    "        sd[str(name)+\" Eff\"] = df_test.iloc[0,0]\n",
    "        sd[str(name)+\" PES\"] = pes(df_test.iloc[0,0])\n",
    "        \n",
    "    else:   \n",
    "        rel = df_test.iloc[0,0]\n",
    "        if rel >= des_prob:\n",
    "            sd[str(name)+\" Rel\"] = rel\n",
    "            sd[str(name)+\" Eff\"] = df_test.iloc[0,1]\n",
    "            sd[str(name)+\" PES\"] = pes(df_test.iloc[0,1])\n",
    "        else:\n",
    "            sd[str(name)+\" Rel\"] = rel\n",
    "            sd[str(name)+\" Eff\"] = \"n/a\"\n",
    "            sd[str(name)+\" PES\"] = \"n/a\"\n",
    "            \n",
    "            \n",
    "    if name == \"PP\":\n",
    "        tp = df_test.columns[0]  # tuned parameter\n",
    "        tp = tp[:7].strip()\n",
    "        sd[\"PP TP\"] = tp\n",
    "        \n",
    "    elif name == \"KM\":\n",
    "        tp = df_test.columns[0]  # tuned parameter\n",
    "        tp = tp[:-4].strip()\n",
    "        sd[\"KM TP\"] = tp\n",
    "    \n",
    "    return sd\n",
    "\n",
    "\n",
    "# fn to run system specific experiments for PP, KM, TM, OR and BL\n",
    "def run_system_specific_experiments(des_prob):\n",
    "    \n",
    "    results = {}\n",
    "    \n",
    "    for des_recall in des_recalls:\n",
    "        df_train = do_pp_method(topics_train, pp_adjusts, des_prob, des_recall)\n",
    "        best_train = get_best_pp_adjust(pp_adjusts, des_prob, des_recall, df_train)\n",
    "        df_test =  do_pp_method(topics_test, best_train, des_prob, des_recall)\n",
    "        sd_pp = get_clean_results_dict(df_test, des_prob, \"PP\")\n",
    "\n",
    "        df_train = do_knee_method(topics_train, km_adjusts, des_recall)\n",
    "        best_train = get_best_knee_adjust(km_adjusts, des_prob, des_recall, df_train)\n",
    "        df_test =  do_knee_method(topics_test, best_train, des_recall)\n",
    "        sd_km = get_clean_results_dict(df_test, des_prob, \"KM\")\n",
    "\n",
    "\n",
    "        df_test =  do_target_method(topics_test, des_recall, des_prob)\n",
    "        sd_tm = get_clean_results_dict(df_test, des_prob, \"TM\")\n",
    "        df_test =  do_baseline_method(topics_test, des_recall)\n",
    "        sd_bl = get_clean_results_dict(df_test, des_prob, \"BL\")\n",
    "        df_test =  do_oracle_method(topics_test, des_recall)\n",
    "        sd_or = get_clean_results_dict(df_test, des_prob, \"OR\")\n",
    "\n",
    "        sd = {}\n",
    "        for k,v in sd_pp.items():\n",
    "            sd[k] = v\n",
    "        for k,v in sd_km.items():\n",
    "            sd[k] = v\n",
    "        for k,v in sd_tm.items():\n",
    "            sd[k] = v\n",
    "        for k,v in sd_bl.items():\n",
    "            sd[k] = v\n",
    "        for k,v in sd_or.items():\n",
    "            sd[k] = v\n",
    "        results[des_recall] = sd\n",
    "\n",
    "    df = pd.DataFrame.from_dict(results).T\n",
    "     \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PP Rel</th>\n",
       "      <th>PP Eff</th>\n",
       "      <th>PP PES</th>\n",
       "      <th>PP TP</th>\n",
       "      <th>KM Rel</th>\n",
       "      <th>KM Eff</th>\n",
       "      <th>KM PES</th>\n",
       "      <th>KM TP</th>\n",
       "      <th>TM Rel</th>\n",
       "      <th>TM Eff</th>\n",
       "      <th>TM PES</th>\n",
       "      <th>BL Rel</th>\n",
       "      <th>BL Eff</th>\n",
       "      <th>BL PES</th>\n",
       "      <th>OR Eff</th>\n",
       "      <th>OR PES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.50</th>\n",
       "      <td>1</td>\n",
       "      <td>22429</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>1</td>\n",
       "      <td>21355</td>\n",
       "      <td>72.6</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>32283</td>\n",
       "      <td>58.6</td>\n",
       "      <td>1</td>\n",
       "      <td>39003</td>\n",
       "      <td>50</td>\n",
       "      <td>2346</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.60</th>\n",
       "      <td>1</td>\n",
       "      <td>22576</td>\n",
       "      <td>71.1</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>1</td>\n",
       "      <td>21355</td>\n",
       "      <td>72.6</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>38184</td>\n",
       "      <td>51.1</td>\n",
       "      <td>1</td>\n",
       "      <td>46803</td>\n",
       "      <td>40</td>\n",
       "      <td>3558</td>\n",
       "      <td>95.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.70</th>\n",
       "      <td>0.95</td>\n",
       "      <td>25064</td>\n",
       "      <td>67.9</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>1</td>\n",
       "      <td>21355</td>\n",
       "      <td>72.6</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>41015</td>\n",
       "      <td>47.4</td>\n",
       "      <td>1</td>\n",
       "      <td>54602</td>\n",
       "      <td>30</td>\n",
       "      <td>5659</td>\n",
       "      <td>92.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.80</th>\n",
       "      <td>1</td>\n",
       "      <td>36559</td>\n",
       "      <td>53.1</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>1</td>\n",
       "      <td>21355</td>\n",
       "      <td>72.6</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>44554</td>\n",
       "      <td>42.9</td>\n",
       "      <td>1</td>\n",
       "      <td>62406</td>\n",
       "      <td>20</td>\n",
       "      <td>7330</td>\n",
       "      <td>90.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.90</th>\n",
       "      <td>0.95</td>\n",
       "      <td>62260</td>\n",
       "      <td>20.2</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>0.85</td>\n",
       "      <td>n/a</td>\n",
       "      <td>n/a</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>54511</td>\n",
       "      <td>30.1</td>\n",
       "      <td>1</td>\n",
       "      <td>70205</td>\n",
       "      <td>10</td>\n",
       "      <td>13292</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.95</th>\n",
       "      <td>0.95</td>\n",
       "      <td>76360</td>\n",
       "      <td>2.1</td>\n",
       "      <td>0.15 5</td>\n",
       "      <td>0.75</td>\n",
       "      <td>n/a</td>\n",
       "      <td>n/a</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>66956</td>\n",
       "      <td>14.2</td>\n",
       "      <td>1</td>\n",
       "      <td>74108</td>\n",
       "      <td>5</td>\n",
       "      <td>17392</td>\n",
       "      <td>77.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     PP Rel PP Eff PP PES   PP TP KM Rel KM Eff KM PES KM TP TM Rel TM Eff  \\\n",
       "0.50      1  22429   71.3  0.15 5      1  21355   72.6    50      1  32283   \n",
       "0.60      1  22576   71.1  0.15 5      1  21355   72.6    50      1  38184   \n",
       "0.70   0.95  25064   67.9  0.15 5      1  21355   72.6    50      1  41015   \n",
       "0.80      1  36559   53.1  0.15 5      1  21355   72.6    50      1  44554   \n",
       "0.90   0.95  62260   20.2  0.15 5   0.85    n/a    n/a    50      1  54511   \n",
       "0.95   0.95  76360    2.1  0.15 5   0.75    n/a    n/a    50      1  66956   \n",
       "\n",
       "     TM PES BL Rel BL Eff BL PES OR Eff OR PES  \n",
       "0.50   58.6      1  39003     50   2346     97  \n",
       "0.60   51.1      1  46803     40   3558   95.4  \n",
       "0.70   47.4      1  54602     30   5659   92.7  \n",
       "0.80   42.9      1  62406     20   7330   90.6  \n",
       "0.90   30.1      1  70205     10  13292     83  \n",
       "0.95   14.2      1  74108      5  17392   77.7  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# run experiments for p = 0.95\n",
    "df_95 = run_system_specific_experiments(0.95)\n",
    "df_95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PP Rel</th>\n",
       "      <th>KM Rel</th>\n",
       "      <th>TM Rel</th>\n",
       "      <th>BL Rel</th>\n",
       "      <th>PP Eff</th>\n",
       "      <th>KM Eff</th>\n",
       "      <th>TM Eff</th>\n",
       "      <th>BL Eff</th>\n",
       "      <th>OR Eff</th>\n",
       "      <th>PP PES</th>\n",
       "      <th>KM PES</th>\n",
       "      <th>TM PES</th>\n",
       "      <th>BL PES</th>\n",
       "      <th>OR PES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.50</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>22429</td>\n",
       "      <td>21355</td>\n",
       "      <td>32283</td>\n",
       "      <td>39003</td>\n",
       "      <td>2346</td>\n",
       "      <td>71.3</td>\n",
       "      <td>72.6</td>\n",
       "      <td>58.6</td>\n",
       "      <td>50</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.60</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>22576</td>\n",
       "      <td>21355</td>\n",
       "      <td>38184</td>\n",
       "      <td>46803</td>\n",
       "      <td>3558</td>\n",
       "      <td>71.1</td>\n",
       "      <td>72.6</td>\n",
       "      <td>51.1</td>\n",
       "      <td>40</td>\n",
       "      <td>95.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.70</th>\n",
       "      <td>0.95</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>25064</td>\n",
       "      <td>21355</td>\n",
       "      <td>41015</td>\n",
       "      <td>54602</td>\n",
       "      <td>5659</td>\n",
       "      <td>67.9</td>\n",
       "      <td>72.6</td>\n",
       "      <td>47.4</td>\n",
       "      <td>30</td>\n",
       "      <td>92.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.80</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>36559</td>\n",
       "      <td>21355</td>\n",
       "      <td>44554</td>\n",
       "      <td>62406</td>\n",
       "      <td>7330</td>\n",
       "      <td>53.1</td>\n",
       "      <td>72.6</td>\n",
       "      <td>42.9</td>\n",
       "      <td>20</td>\n",
       "      <td>90.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.90</th>\n",
       "      <td>0.95</td>\n",
       "      <td>0.85</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>62260</td>\n",
       "      <td>n/a</td>\n",
       "      <td>54511</td>\n",
       "      <td>70205</td>\n",
       "      <td>13292</td>\n",
       "      <td>20.2</td>\n",
       "      <td>n/a</td>\n",
       "      <td>30.1</td>\n",
       "      <td>10</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.95</th>\n",
       "      <td>0.95</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>76360</td>\n",
       "      <td>n/a</td>\n",
       "      <td>66956</td>\n",
       "      <td>74108</td>\n",
       "      <td>17392</td>\n",
       "      <td>2.1</td>\n",
       "      <td>n/a</td>\n",
       "      <td>14.2</td>\n",
       "      <td>5</td>\n",
       "      <td>77.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     PP Rel KM Rel TM Rel BL Rel PP Eff KM Eff TM Eff BL Eff OR Eff PP PES  \\\n",
       "0.50      1      1      1      1  22429  21355  32283  39003   2346   71.3   \n",
       "0.60      1      1      1      1  22576  21355  38184  46803   3558   71.1   \n",
       "0.70   0.95      1      1      1  25064  21355  41015  54602   5659   67.9   \n",
       "0.80      1      1      1      1  36559  21355  44554  62406   7330   53.1   \n",
       "0.90   0.95   0.85      1      1  62260    n/a  54511  70205  13292   20.2   \n",
       "0.95   0.95   0.75      1      1  76360    n/a  66956  74108  17392    2.1   \n",
       "\n",
       "     KM PES TM PES BL PES OR PES  \n",
       "0.50   72.6   58.6     50     97  \n",
       "0.60   72.6   51.1     40   95.4  \n",
       "0.70   72.6   47.4     30   92.7  \n",
       "0.80   72.6   42.9     20   90.6  \n",
       "0.90    n/a   30.1     10     83  \n",
       "0.95    n/a   14.2      5   77.7  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clean df for paper inclusion\n",
    "df_95 = df_95.drop(\"PP TP\", axis = 1)\n",
    "df_95 = df_95.drop(\"KM TP\", axis = 1)\n",
    "cols = ['PP Rel', 'KM Rel','TM Rel', 'BL Rel','PP Eff','KM Eff',  'TM Eff',  \n",
    "        'BL Eff', 'OR Eff',  'PP PES', 'KM PES','TM PES', \"BL PES\",'OR PES']\n",
    "df_95 = df_95[cols]\n",
    "df_95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllllllllllll}\n",
      "\\toprule\n",
      "{} & PP Rel & KM Rel & TM Rel & BL Rel & PP Eff & KM Eff & TM Eff & BL Eff & OR Eff & PP PES & KM PES & TM PES & BL PES & OR PES \\\\\n",
      "\\midrule\n",
      "0.50 &      1 &      1 &      1 &      1 &  22429 &  21355 &  32283 &  39003 &   2346 &   71.3 &   72.6 &   58.6 &     50 &     97 \\\\\n",
      "0.60 &      1 &      1 &      1 &      1 &  22576 &  21355 &  38184 &  46803 &   3558 &   71.1 &   72.6 &   51.1 &     40 &   95.4 \\\\\n",
      "0.70 &   0.95 &      1 &      1 &      1 &  25064 &  21355 &  41015 &  54602 &   5659 &   67.9 &   72.6 &   47.4 &     30 &   92.7 \\\\\n",
      "0.80 &      1 &      1 &      1 &      1 &  36559 &  21355 &  44554 &  62406 &   7330 &   53.1 &   72.6 &   42.9 &     20 &   90.6 \\\\\n",
      "0.90 &   0.95 &   0.85 &      1 &      1 &  62260 &    n/a &  54511 &  70205 &  13292 &   20.2 &    n/a &   30.1 &     10 &     83 \\\\\n",
      "0.95 &   0.95 &   0.75 &      1 &      1 &  76360 &    n/a &  66956 &  74108 &  17392 &    2.1 &    n/a &   14.2 &      5 &   77.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df_95.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run experiments for p = 0.7\n",
    "df_70 = run_system_specific_experiments(0.7)\n",
    "df_70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean df for paper inclusion\n",
    "df_70 = df_70.drop(\"PP TP\", axis = 1)\n",
    "df_70 = df_70.drop(\"KM TP\", axis = 1)\n",
    "cols = ['PP Rel', 'KM Rel','TM Rel', 'BL Rel','PP Eff','KM Eff',  'TM Eff',  \n",
    "        'BL Eff', 'OR Eff',  'PP PES', 'KM PES','TM PES', \"BL PES\",'OR PES']\n",
    "df_70 = df_70[cols]\n",
    "df_70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_70.to_latex())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}