scm-sampler-duplicate-lists.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess as sp\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns  # for aesthetic\n",
    "%matplotlib inline\n",
    "sns.set_style('ticks')\n",
    "import tqdm \n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from simplex_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: ./tmp: File exists\r\n"
     ]
    }
   ],
   "source": [
    "!mkdir ./tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def facet_list_to_graph(facet_list):\n",
    "    \"\"\"Convert a facet list to a bipartite graph\"\"\"\n",
    "    g = nx.Graph()\n",
    "    for f, facet in enumerate(facet_list):\n",
    "        for v in facet:\n",
    "            g.add_edge('v' + str(v), 'f' + str(f))  # differentiate node types\n",
    "    return g\n",
    "\n",
    "def seq_to_file(nums,f):\n",
    "    with open(f, mode='wb', encoding='utf-8') as myfile:\n",
    "        myfile.write(' '.join(map(str,nums)));\n",
    "        \n",
    "\n",
    "def facets_to_file(cliques,f):\n",
    "    with open(f, mode='w', encoding='utf-8') as myfile:\n",
    "        for clique in cliques:\n",
    "            myfile.write(' '.join(str(el) for el in clique))\n",
    "            myfile.write('\\n')       \n",
    "            \n",
    "def sanitize_sequence(sequence):\n",
    "    count = 0;\n",
    "    relabel = {}\n",
    "    new_cliques = []\n",
    "    for clique in sequence:\n",
    "        for el in clique:\n",
    "            if el not in relabel:\n",
    "                relabel[el] = count\n",
    "                count+=1;\n",
    "        new_cliques.append(list(map(lambda x: relabel[x], clique)));   \n",
    "    return new_cliques;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rejection_sampling(command, seed=0):\n",
    "    # Call sampler with subprocess\n",
    "    proc = sp.run(command, stdout=sp.PIPE)\n",
    "    # Read output as a facet list \n",
    "    facet_list = []\n",
    "    for line in proc.stdout.decode().split(\"\\n\")[1:-1]:\n",
    "        if line.find(\"#\") == 0:\n",
    "            yield facet_list\n",
    "            facet_list = []\n",
    "        else:\n",
    "            facet_list.append([int(x) for x in line.strip().split()])\n",
    "    yield facet_list\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiply_cliques(cliques, n):\n",
    "    s = []\n",
    "    sc = sanitize_sequence(cliques);\n",
    "    for el in sc:\n",
    "        s.extend(el);\n",
    "    m = np.max(list(set(s)))\n",
    "    new_cliques = []\n",
    "    new_cliques.extend(sc);\n",
    "    for nn in range(1,n):\n",
    "        new_cliques.extend(map(lambda x: list(np.array(x)+nn*m), sc));\n",
    "    return new_cliques;\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e9371ca0cfb441b81fd430be37c5cc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd35228a523d41ca840233a278706d7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10d34db797a34a72bae9ffc47614bb3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "219a22be5e0d4c52915996df639e0944",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7360c4e55e24418a449df1b96e5412a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a7e66ac37ae4e85bee5c3af65f7515b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "datasetlist = ['InVS13','InVS15','LH10','SFHH','LyonSchool'] #['Thiers13'] \n",
    "nms = [5,10]#,15,30]\n",
    "thrs = [0.8] # [0.95, 0.9, 0.85, 0.8]\n",
    "thr_dir = 'thr_data/'\n",
    "output_dir = 'thr_data_duplicate/'\n",
    "iterations = 10;\n",
    "n = 5;\n",
    "\n",
    "from random import seed, randint\n",
    "seed(int(time()));\n",
    "\n",
    "for dataset in tqdm_notebook(datasetlist):\n",
    "    for nm in tqdm_notebook(nms):\n",
    "        for thr in thrs:\n",
    "            filename = thr_dir+'aggr_'+str(nm)+'_'+str(thr)+'min_cliques_'+dataset+'.json';\n",
    "            cliques = json.load(open(filename));\n",
    "            sc = sanitize_sequence(cliques);\n",
    "            sc = multiply_cliques(sc, n);\n",
    "            facets_to_file(sc, './tmp/facet_list.txt')\n",
    "            command = ['scm/bin/mcmc_sampler', './tmp/facet_list.txt', '-t', str(iterations),\n",
    "                    '-c', '-f', '10000'] #'--seed='+str(randint(0,10000000)),'-b', '500', '-f', '500'\n",
    "            jd = open(output_dir+'random_'+str(nm)+'_'+str(thr)+'min_cliques_'+dataset+'.json','w')\n",
    "            ls = []\n",
    "            for facet_list in rejection_sampling(command):\n",
    "                ls.append(facet_list)\n",
    "            json.dump(ls,jd)\n",
    "            jd.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deh = json.load(open('./thr_data_duplicate/random_15_0.8min_cliques_Thiers13.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deh[0][:5], deh[1][:5]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}