DMN+.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOADING PREPROCESSED DATA\n",
    "\n",
    "Loading GloVe word embeddings. Building functions to convert words into their vector representations and vice versa. Loading babi induction task 10K dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from __future__ import division\n",
    "\n",
    "filename = 'glove.6B.100d.txt'\n",
    "\n",
    "def loadEmbeddings(filename):\n",
    "    vocab = []\n",
    "    embd = []\n",
    "    file = open(filename,'r')\n",
    "    for line in file.readlines():\n",
    "        row = line.strip().split(' ')\n",
    "        vocab.append(row[0])\n",
    "        embd.append(row[1:])\n",
    "    print('Loaded!')\n",
    "    file.close()\n",
    "    return vocab,embd\n",
    "vocab,embd = loadEmbeddings(filename)\n",
    "\n",
    "\n",
    "word_vec_dim = len(embd[0])\n",
    "\n",
    "vocab.append('<UNK>')\n",
    "embd.append(np.asarray(embd[vocab.index('unk')],np.float32)+0.01)\n",
    "\n",
    "vocab.append('<EOS>')\n",
    "embd.append(np.asarray(embd[vocab.index('eos')],np.float32)+0.01)\n",
    "\n",
    "vocab.append('<PAD>')\n",
    "embd.append(np.zeros((word_vec_dim),np.float32))\n",
    "\n",
    "embedding = np.asarray(embd)\n",
    "embedding = embedding.astype(np.float32)\n",
    "\n",
    "def word2vec(word):  # converts a given word into its vector representation\n",
    "    if word in vocab:\n",
    "        return embedding[vocab.index(word)]\n",
    "    else:\n",
    "        return embedding[vocab.index('<UNK>')]\n",
    "\n",
    "def most_similar_eucli(x):\n",
    "    xminusy = np.subtract(embedding,x)\n",
    "    sq_xminusy = np.square(xminusy)\n",
    "    sum_sq_xminusy = np.sum(sq_xminusy,1)\n",
    "    eucli_dists = np.sqrt(sum_sq_xminusy)\n",
    "    return np.argsort(eucli_dists)\n",
    "\n",
    "def vec2word(vec):   # converts a given vector representation into the represented word \n",
    "    most_similars = most_similar_eucli(np.asarray(vec,np.float32))\n",
    "    return vocab[most_similars[0]]\n",
    "\n",
    "import pickle\n",
    "\n",
    "with open ('embeddingPICKLE', 'rb') as fp:\n",
    "    processed_data = pickle.load(fp)\n",
    "\n",
    "fact_stories = processed_data[0]\n",
    "questions = processed_data[1]\n",
    "answers = np.reshape(processed_data[2],(len(processed_data[2])))\n",
    "test_fact_stories = processed_data[3]\n",
    "test_questions = processed_data[4]\n",
    "test_answers = np.reshape(processed_data[5],(len(processed_data[5])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EXAMPLE DATA:\n",
      "\n",
      "FACTS:\n",
      "\n",
      "1)  ['julius', 'is', 'a', 'swan']\n",
      "2)  ['greg', 'is', 'a', 'rhino']\n",
      "3)  ['brian', 'is', 'a', 'frog']\n",
      "4)  ['julius', 'is', 'white', '<PAD>']\n",
      "5)  ['lily', 'is', 'a', 'rhino']\n",
      "6)  ['lily', 'is', 'yellow', '<PAD>']\n",
      "7)  ['greg', 'is', 'yellow', '<PAD>']\n",
      "8)  ['bernhard', 'is', 'a', 'frog']\n",
      "9)  ['brian', 'is', 'green', '<PAD>']\n",
      "\n",
      "QUESTION:\n",
      "['what', 'color', 'is', 'bernhard']\n",
      "\n",
      "ANSWER:\n",
      "green\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "print \"EXAMPLE DATA:\\n\"\n",
    "\n",
    "sample = random.randint(0,len(fact_stories))\n",
    "\n",
    "print \"FACTS:\\n\"\n",
    "for i in xrange(len(fact_stories[sample])):\n",
    "    print str(i+1)+\") \",\n",
    "    print map(vec2word,fact_stories[sample][i])\n",
    "    \n",
    "print \"\\nQUESTION:\"\n",
    "print map(vec2word,questions[sample])\n",
    "print \"\\nANSWER:\"\n",
    "print vocab[answers[sample]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CREATING TRAINING AND VALIDATION DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "train_fact_stories = []\n",
    "train_questions = []\n",
    "train_answers = []\n",
    "val_fact_stories = []\n",
    "val_questions = []\n",
    "val_answers = []\n",
    "\n",
    "p=90 #(90% data used for training. Rest for validation)\n",
    "    \n",
    "train_len = int((p/100)*len(fact_stories))\n",
    "\n",
    "train_fact_stories = fact_stories[0:train_len] \n",
    "val_fact_stories = fact_stories[train_len:]\n",
    "\n",
    "train_questions = questions[0:train_len] \n",
    "val_questions = questions[train_len:] \n",
    "\n",
    "train_answers = answers[0:train_len] \n",
    "val_answers = answers[train_len:] \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SENTENCE READING LAYER IMPLEMENTED BEFOREHAND \n",
    "\n",
    "Positionally encode the word vectors in each sentence, and combine all the words in the sentence to create a fixed sized vector representation for the sentence.\n",
    "\n",
    "\"sentence embedding\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from __future__ import division\n",
    "\n",
    "def sentence_reader(fact_stories): #positional_encoder\n",
    "    \n",
    "    PAD_val = np.zeros((word_vec_dim),np.float32)\n",
    "    \n",
    "    pe_fact_stories = np.zeros((fact_stories.shape[0],fact_stories.shape[1],word_vec_dim),np.float32)\n",
    "    \n",
    "    for fact_story_index in xrange(0,len(fact_stories)):\n",
    "        for fact_index in xrange(0,len(fact_stories[fact_story_index])):\n",
    "            \n",
    "            M = 0\n",
    "            \n",
    "            # Code to ignore pads. \n",
    "            for word_position in xrange(len(fact_stories[fact_story_index,fact_index])):\n",
    "                if np.all(np.equal(PAD_val,fact_stories[fact_story_index,fact_index,word_position])):\n",
    "                    break\n",
    "                else:\n",
    "                    M+=1\n",
    "                \n",
    "            l = np.zeros((word_vec_dim),np.float32) \n",
    "            \n",
    "            # ljd = (1 − j/M) − (d/D)(1 − 2j/M),\n",
    "            \n",
    "            for word_position in xrange(0,M):\n",
    "                \n",
    "                for dimension in xrange(0,word_vec_dim):\n",
    "                    \n",
    "                    j = word_position + 1 # making position start from 1 instead of 0\n",
    "                    d = dimension + 1 # making dimensions start from 1 isntead of 0 (1-100 instead of 0-99)\n",
    "                    \n",
    "                    l[dimension] = (1-(j/M)) - (d/word_vec_dim)*(1-2*(j/M))\n",
    " \n",
    "                \n",
    "                pe_fact_stories[fact_story_index,fact_index] += np.multiply(l,fact_stories[fact_story_index,fact_index,word_position])\n",
    "\n",
    "\n",
    "    return pe_fact_stories\n",
    "\n",
    "train_fact_stories = sentence_reader(train_fact_stories)\n",
    "val_fact_stories = sentence_reader(val_fact_stories)\n",
    "test_fact_stories = sentence_reader(test_fact_stories)\n",
    "                \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to create randomized batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_batches(fact_stories,questions,answers,batch_size):\n",
    "    \n",
    "    shuffle = np.arange(len(questions))\n",
    "    np.random.shuffle(shuffle)\n",
    "    \n",
    "    batches_fact_stories = []\n",
    "    batches_questions = []\n",
    "    batches_answers = []\n",
    "    \n",
    "    i=0\n",
    "    \n",
    "    while i+batch_size<=len(questions):\n",
    "        batch_fact_stories = []\n",
    "        batch_questions = []\n",
    "        batch_answers = []\n",
    "        \n",
    "        for j in xrange(i,i+batch_size):\n",
    "            batch_fact_stories.append(fact_stories[shuffle[j]])\n",
    "            batch_questions.append(questions[shuffle[j]])\n",
    "            batch_answers.append(answers[shuffle[j]])\n",
    "            \n",
    "        batch_fact_stories = np.asarray(batch_fact_stories,np.float32)\n",
    "        batch_fact_stories = np.transpose(batch_fact_stories,[1,0,2])\n",
    "        #result = number of facts x batch_size x fact sentence size x word vector size\n",
    "        \n",
    "        batch_questions = np.asarray(batch_questions,np.float32)\n",
    "        batch_questions = np.transpose(batch_questions,[1,0,2])\n",
    "        #result = question_length x batch_size x fact sentence size x word vector size\n",
    "        \n",
    "        batches_fact_stories.append(batch_fact_stories)\n",
    "        batches_questions.append(batch_questions)\n",
    "        batches_answers.append(batch_answers)\n",
    "        \n",
    "        i+=batch_size\n",
    "        \n",
    "    batches_fact_stories = np.asarray(batches_fact_stories,np.float32)\n",
    "    batches_questions = np.asarray(batches_questions,np.float32)\n",
    "    batches_answers = np.asarray(batches_answers,np.int32)\n",
    "    \n",
    "    return batches_fact_stories,batches_questions,batches_answers\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Tensorflow placeholders\n",
    "\n",
    "tf_facts = tf.placeholder(tf.float32,[None,None,word_vec_dim])\n",
    "tf_questions = tf.placeholder(tf.float32,[None,None,word_vec_dim])\n",
    "tf_answers = tf.placeholder(tf.int32,[None])\n",
    "training = tf.placeholder(tf.bool)\n",
    "\n",
    "#hyperparameters\n",
    "epochs = 256\n",
    "learning_rate = 0.001\n",
    "hidden_size = 100\n",
    "passes = 3\n",
    "dropout_rate = 0.1\n",
    "beta = 0.0001 #l2 regularization scale\n",
    "\n",
    "regularizer = tf.contrib.layers.l2_regularizer(scale=beta) #l2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All the trainable parameters initialized here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "\n",
    "# FORWARD GRU PARAMETERS FOR INPUT MODULE\n",
    "\n",
    "wf = tf.get_variable(\"wf\", shape=[3,word_vec_dim, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer= regularizer)\n",
    "uf = tf.get_variable(\"uf\", shape=[3,hidden_size, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer=regularizer)\n",
    "bf = tf.get_variable(\"bf\", shape=[3,hidden_size],initializer=tf.zeros_initializer())\n",
    "\n",
    "\n",
    "# BACKWARD GRU PARAMETERS FOR INPUT MODULE\n",
    "\n",
    "wb = tf.get_variable(\"wb\", shape=[3,word_vec_dim, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer=regularizer)\n",
    "ub = tf.get_variable(\"ub\", shape=[3,hidden_size, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer=regularizer)\n",
    "bb = tf.get_variable(\"bb\", shape=[3,hidden_size],initializer=tf.zeros_initializer())\n",
    "\n",
    "# GRU PARAMETERS FOR QUESTION MODULE (TO ENCODE THE QUESTIONS)\n",
    "\n",
    "wq = tf.get_variable(\"wq\", shape=[3,word_vec_dim, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer=regularizer)\n",
    "uq = tf.get_variable(\"uq\", shape=[3,hidden_size, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                      regularizer=regularizer)\n",
    "bq = tf.get_variable(\"bq\", shape=[3,hidden_size],initializer=tf.zeros_initializer())\n",
    "\n",
    "\n",
    "# EPISODIC MEMORY\n",
    "\n",
    "# ATTENTION MECHANISM\n",
    "\n",
    "inter_neurons = hidden_size\n",
    "\n",
    "w1 = tf.get_variable(\"w1\", shape=[hidden_size*4, inter_neurons],\n",
    "                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                     regularizer=regularizer)\n",
    "b1 = tf.get_variable(\"b1\", shape=[inter_neurons],\n",
    "                     initializer=tf.zeros_initializer())\n",
    "w2 = tf.get_variable(\"w2\", shape=[inter_neurons,1],\n",
    "                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                     regularizer=regularizer)\n",
    "b2 = tf.get_variable(\"b2\", shape=[1],initializer=tf.zeros_initializer())\n",
    "\n",
    "\n",
    "# ATTENTION BASED GRU PARAMETERS\n",
    "\n",
    "watt = tf.get_variable(\"watt\", shape=[2,hidden_size,hidden_size],\n",
    "                       initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                       regularizer=regularizer)\n",
    "uatt = tf.get_variable(\"uatt\", shape=[2,hidden_size, hidden_size],\n",
    "                      initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                       regularizer=regularizer)\n",
    "batt = tf.get_variable(\"batt\", shape=[2,hidden_size],initializer=tf.zeros_initializer())\n",
    "\n",
    "\n",
    "# MEMORY UPDATE PARAMETERS\n",
    "# (UNTIED)\n",
    "\n",
    "wt = tf.get_variable(\"wt\", shape=[passes,hidden_size*3,hidden_size],\n",
    "                    initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                    regularizer=regularizer)\n",
    "bt = tf.get_variable(\"bt\", shape=[passes,hidden_size],\n",
    "                     initializer=tf.zeros_initializer())\n",
    "\n",
    "# ANSWER MODULE PARAMETERS\n",
    "\n",
    "wa_pd = tf.get_variable(\"wa_pd\", shape=[hidden_size*2,len(vocab)],\n",
    "                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                     regularizer=regularizer)\n",
    "ba_pd = tf.get_variable(\"ba_pd\", shape=[len(vocab)],\n",
    "                     initializer=tf.zeros_initializer())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Layer Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def layer_norm(inputs,scope,scale=True,layer_norm=True,epsilon = 1e-5):\n",
    "    \n",
    "    if layer_norm == True:\n",
    "        \n",
    "        with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n",
    "            \n",
    "            if scale == False:\n",
    "                scale = tf.ones([inputs.get_shape()[1]],tf.float32)\n",
    "            else:\n",
    "                scale = tf.get_variable(\"scale\", shape=[inputs.get_shape()[1]],\n",
    "                        initializer=tf.ones_initializer())\n",
    "        \n",
    "        \n",
    "        ## ignored shift - bias will be externally added which can produce shift\n",
    "\n",
    "        mean, var = tf.nn.moments(inputs, [1], keep_dims=True)\n",
    "        \n",
    "        LN = tf.multiply((scale / tf.sqrt(var + epsilon)),(inputs - mean))\n",
    "        \n",
    "        return LN\n",
    "    else:\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  GRU Function\n",
    "\n",
    "Returns a tensor of all the hidden states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GRU(inp,hidden,\n",
    "        w,u,b,\n",
    "        seq_len,scope):\n",
    "\n",
    "    hidden_lists = tf.TensorArray(size=seq_len,dtype=tf.float32)\n",
    "    \n",
    "    i=0\n",
    "    \n",
    "    def cond(i,hidden,hidden_lists):\n",
    "        return i < seq_len\n",
    "    \n",
    "    def body(i,hidden,hidden_lists):\n",
    "        \n",
    "        x = inp[i]\n",
    " \n",
    "        # GRU EQUATIONS:\n",
    "        z = tf.sigmoid(layer_norm( tf.matmul(x,w[0]) + tf.matmul(hidden,u[0]) , scope+\"_z\")+ b[0])\n",
    "        r = tf.sigmoid(layer_norm( tf.matmul(x,w[1]) + tf.matmul(hidden,u[1]) , scope+\"_r\")+ b[1])\n",
    "        h_ = tf.tanh(layer_norm( tf.matmul(x,w[2]) + tf.multiply(r,tf.matmul(hidden,u[2])),scope+\"_h\") + b[2])\n",
    "        hidden = tf.multiply(z,h_) + tf.multiply((1-z),hidden)\n",
    "\n",
    "        hidden_lists = hidden_lists.write(i,hidden)\n",
    "        \n",
    "        return i+1,hidden,hidden_lists\n",
    "    \n",
    "    _,_,hidden_lists = tf.while_loop(cond,body,[i,hidden,hidden_lists])\n",
    "    \n",
    "    return hidden_lists.stack()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention based GRU\n",
    "\n",
    "Returns only the final hidden state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention_based_GRU(inp,hidden,\n",
    "                        w,u,b,\n",
    "                        g,seq_len,scope):\n",
    "    \n",
    "    i=0\n",
    "    \n",
    "    def cond(i,hidden):\n",
    "        return i < seq_len\n",
    "    \n",
    "    def body(i,hidden):\n",
    "        \n",
    "        x = inp[i]\n",
    "\n",
    "        # GRU EQUATIONS:\n",
    "        r = tf.sigmoid(layer_norm( tf.matmul(x,w[0]) + tf.matmul(hidden,u[0]), scope+\"_r\") + b[0])\n",
    "        h_ = tf.tanh(layer_norm( tf.matmul(x,w[1]) + tf.multiply(r,tf.matmul(hidden,u[1])),scope+\"_h\") + b[1])\n",
    "        hidden = tf.multiply(g[i],h_) + tf.multiply((1-g[i]),hidden)\n",
    "        \n",
    "        return i+1,hidden\n",
    "    \n",
    "    _,hidden = tf.while_loop(cond,body,[i,hidden])\n",
    "    \n",
    "    return hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dynamic Memory Network+ Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DMN_plus(tf_facts,tf_questions):\n",
    "    \n",
    "    facts_num = tf.shape(tf_facts)[0]\n",
    "    tf_batch_size = tf.shape(tf_questions)[1]\n",
    "    question_len = tf.shape(tf_questions)[0]\n",
    "    \n",
    "    hidden = tf.zeros([tf_batch_size,hidden_size],tf.float32)\n",
    "\n",
    "    # Input Module\n",
    "\n",
    "    # input fusion layer \n",
    "    # bidirectional GRU\n",
    "    \n",
    "    forward = GRU(tf_facts,hidden,\n",
    "                  wf,uf,bf,\n",
    "                  facts_num,\"Forward_GRU\")\n",
    "    \n",
    "    backward = GRU(tf.reverse(tf_facts,[0]),hidden,\n",
    "                   wb,ub,bb,\n",
    "                   facts_num,\"Backward_GRU\")\n",
    "    \n",
    "    backward = tf.reverse(backward,[0])\n",
    "    \n",
    "    encoded_input = tf.add(forward,backward)\n",
    "    \n",
    "    # encoded input now shape = facts_num x batch_size x hidden_size\n",
    "    \n",
    "    encoded_input = tf.layers.dropout(encoded_input,dropout_rate,training=training)\n",
    "\n",
    "    # Question Module\n",
    "    \n",
    "    question_representation = GRU(tf_questions,hidden,\n",
    "                                  wq,uq,bq,\n",
    "                                  question_len,\"Question_GRU\")\n",
    "    \n",
    "    #question_representation's current shape = question len x batch size x hidden size\n",
    "    \n",
    "    question_representation = question_representation[question_len-1]\n",
    "    \n",
    "    #^we will only use the final hidden state. \n",
    "\n",
    "    question_representation = tf.reshape(question_representation,[tf_batch_size,1,hidden_size])\n",
    "    \n",
    "    # Episodic Memory Module\n",
    "    \n",
    "    episodic_memory = tf.identity(question_representation)\n",
    "    \n",
    "    encoded_input = tf.transpose(encoded_input,[1,0,2])\n",
    "    #now shape = batch_size x facts_num x hidden_size\n",
    "    \n",
    "\n",
    "    for i in xrange(passes):\n",
    "        \n",
    "        # Attention Mechanism\n",
    "        \n",
    "        Z1 = tf.multiply(encoded_input,question_representation)\n",
    "        Z2 = tf.multiply(encoded_input,episodic_memory)\n",
    "        Z3 = tf.abs(tf.subtract(encoded_input,question_representation))\n",
    "        Z4 = tf.abs(tf.subtract(encoded_input,episodic_memory))\n",
    "        \n",
    "        Z = tf.concat([Z1,Z2,Z3,Z4],2)\n",
    "        \n",
    "        Z = tf.reshape(Z,[-1,4*hidden_size])\n",
    "        Z = tf.matmul( tf.tanh( layer_norm( tf.matmul(Z,w1), \"Attention_Mechanism\") + b1),w2 ) \n",
    "        Z = layer_norm(Z,\"Attention_Mechanism_2\") + b2\n",
    "        Z = tf.reshape(Z,[tf_batch_size,1,facts_num])\n",
    "        \n",
    "        g = tf.nn.softmax(Z)\n",
    "\n",
    "        g = tf.transpose(g,[2,0,1])\n",
    "        \n",
    "        context_vector = attention_based_GRU(tf.transpose(encoded_input,[1,0,2]),\n",
    "                                             hidden,\n",
    "                                             watt,uatt,batt,\n",
    "                                             g,facts_num,\"Attention_GRU\")\n",
    "                                             \n",
    "        \n",
    "        context_vector = tf.reshape(context_vector,[tf_batch_size,1,hidden_size])\n",
    "        \n",
    "        # Episodic Memory Update\n",
    "        \n",
    "        concated = tf.concat([episodic_memory,context_vector,question_representation],2)\n",
    "        concated = tf.reshape(concated,[-1,3*hidden_size])\n",
    "        \n",
    "        episodic_memory = tf.nn.relu(layer_norm(tf.matmul(concated,wt[i]),\"Memory_Update\",scale=False) + bt[i])\n",
    "        \n",
    "        episodic_memory = tf.reshape(episodic_memory,[tf_batch_size,1,hidden_size])\n",
    "\n",
    "    # Answer module \n",
    "    \n",
    "    # (single word answer prediction)\n",
    "\n",
    "    episodic_memory = tf.reshape(episodic_memory,[tf_batch_size,hidden_size])\n",
    "    episodic_memory = tf.layers.dropout(episodic_memory,dropout_rate,training=training)\n",
    "\n",
    "    question_representation = tf.reshape(question_representation,[tf_batch_size,hidden_size])\n",
    "    \n",
    "    y_concat = tf.concat([question_representation,episodic_memory],1)\n",
    "    \n",
    "    # Convert to pre-softmax probability distribution\n",
    "    y = tf.matmul(y_concat,wa_pd) + ba_pd\n",
    "    return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost function, Evaluation, Optimization function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_output = DMN_plus(tf_facts,tf_questions)\n",
    "\n",
    "\n",
    "# l2 regularization\n",
    "reg_variables = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "regularization = tf.contrib.layers.apply_regularization(regularizer, tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))\n",
    "\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model_output, \n",
    "                                                                     labels=tf_answers))+regularization\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "model_output = tf.nn.softmax(model_output)\n",
    "\n",
    "#Evaluate model\n",
    "correct_pred = tf.equal(tf.cast(tf.argmax(model_output,1),tf.int32),tf_answers)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 0, Loss= 12.902, Accuracy= 0.000\n",
      "Iter 20, Loss= 8.073, Accuracy= 19.531\n",
      "Iter 40, Loss= 2.768, Accuracy= 21.875\n",
      "Iter 60, Loss= 1.457, Accuracy= 19.531\n",
      "\n",
      "Epoch 1, Validation Loss= 1.420, validation Accuracy= 27.000%\n",
      "Epoch 1, Average Training Loss= 5.376, Average Training Accuracy= 24.442%\n",
      "Checkpoint created!\n",
      "\n",
      "Iter 0, Loss= 1.423, Accuracy= 22.656\n",
      "Iter 20, Loss= 1.411, Accuracy= 25.000\n",
      "Iter 40, Loss= 1.404, Accuracy= 25.781\n",
      "Iter 60, Loss= 1.402, Accuracy= 25.000\n",
      "\n",
      "Epoch 2, Validation Loss= 1.395, validation Accuracy= 34.600%\n",
      "Epoch 2, Average Training Loss= 1.408, Average Training Accuracy= 25.413%\n",
      "Checkpoint created!\n",
      "\n",
      "Iter 0, Loss= 1.394, Accuracy= 37.500\n",
      "Iter 20, Loss= 1.352, Accuracy= 46.875\n",
      "Iter 40, Loss= 1.309, Accuracy= 33.594\n",
      "Iter 60, Loss= 1.234, Accuracy= 50.000\n",
      "\n",
      "Epoch 3, Validation Loss= 1.238, validation Accuracy= 44.300%\n",
      "Epoch 3, Average Training Loss= 1.330, Average Training Accuracy= 37.891%\n",
      "Checkpoint created!\n",
      "\n",
      "Iter 0, Loss= 1.224, Accuracy= 46.094\n",
      "Iter 20, Loss= 1.164, Accuracy= 46.875\n",
      "Iter 40, Loss= 1.116, Accuracy= 41.406\n",
      "Iter 60, Loss= 1.019, Accuracy= 58.594\n",
      "\n",
      "Epoch 4, Validation Loss= 1.043, validation Accuracy= 48.500%\n",
      "Epoch 4, Average Training Loss= 1.129, Average Training Accuracy= 47.600%\n",
      "Checkpoint created!\n",
      "\n",
      "Iter 0, Loss= 1.065, Accuracy= 46.094\n",
      "Iter 20, Loss= 1.054, Accuracy= 43.750\n",
      "Iter 40, Loss= 1.027, Accuracy= 45.312\n",
      "Iter 60, Loss= 0.969, Accuracy= 58.594\n",
      "\n",
      "Epoch 5, Validation Loss= 0.991, validation Accuracy= 47.500%\n",
      "Epoch 5, Average Training Loss= 1.013, Average Training Accuracy= 48.583%\n",
      "\n",
      "Iter 0, Loss= 0.957, Accuracy= 47.656\n",
      "Iter 20, Loss= 1.033, Accuracy= 42.188\n",
      "Iter 40, Loss= 1.015, Accuracy= 46.875\n",
      "Iter 60, Loss= 0.989, Accuracy= 45.312\n",
      "\n",
      "Epoch 6, Validation Loss= 0.939, validation Accuracy= 50.700%\n",
      "Epoch 6, Average Training Loss= 0.975, Average Training Accuracy= 48.426%\n",
      "Checkpoint created!\n",
      "\n",
      "Iter 0, Loss= 0.942, Accuracy= 50.781\n",
      "Iter 20, Loss= 0.963, Accuracy= 46.875\n",
      "Iter 40, Loss= 0.954, Accuracy= 42.188\n",
      "Iter 60, Loss= 0.920, Accuracy= 53.125\n",
      "\n",
      "Epoch 7, Validation Loss= 0.945, validation Accuracy= 49.200%\n",
      "Epoch 7, Average Training Loss= 0.942, Average Training Accuracy= 48.393%\n",
      "\n",
      "Iter 0, Loss= 0.935, Accuracy= 53.906\n",
      "Iter 20, Loss= 0.921, Accuracy= 48.438\n",
      "Iter 40, Loss= 0.951, Accuracy= 51.562\n",
      "Iter 60, Loss= 0.955, Accuracy= 53.125\n",
      "\n",
      "Epoch 8, Validation Loss= 0.920, validation Accuracy= 50.100%\n",
      "Epoch 8, Average Training Loss= 0.921, Average Training Accuracy= 49.007%\n",
      "\n",
      "Iter 0, Loss= 0.873, Accuracy= 50.781\n",
      "Iter 20, Loss= 0.878, Accuracy= 53.125\n",
      "Iter 40, Loss= 1.016, Accuracy= 42.969\n",
      "Iter 60, Loss= 0.916, Accuracy= 50.000\n",
      "\n",
      "Epoch 9, Validation Loss= 0.938, validation Accuracy= 47.800%\n",
      "Epoch 9, Average Training Loss= 0.911, Average Training Accuracy= 49.576%\n",
      "\n",
      "Iter 0, Loss= 0.876, Accuracy= 53.125\n",
      "Iter 20, Loss= 0.906, Accuracy= 51.562\n",
      "Iter 40, Loss= 0.885, Accuracy= 52.344\n",
      "Iter 60, Loss= 0.872, Accuracy= 43.750\n",
      "\n",
      "Epoch 10, Validation Loss= 0.915, validation Accuracy= 47.200%\n",
      "Epoch 10, Average Training Loss= 0.905, Average Training Accuracy= 48.940%\n",
      "\n",
      "Iter 0, Loss= 0.926, Accuracy= 45.312\n",
      "Iter 20, Loss= 0.900, Accuracy= 48.438\n",
      "Iter 40, Loss= 0.960, Accuracy= 47.656\n",
      "Iter 60, Loss= 0.870, Accuracy= 53.125\n",
      "\n",
      "Epoch 11, Validation Loss= 0.906, validation Accuracy= 49.100%\n",
      "Epoch 11, Average Training Loss= 0.895, Average Training Accuracy= 49.799%\n",
      "\n",
      "Iter 0, Loss= 0.864, Accuracy= 57.031\n",
      "Iter 20, Loss= 0.846, Accuracy= 58.594\n",
      "Iter 40, Loss= 0.912, Accuracy= 42.969\n",
      "Iter 60, Loss= 0.872, Accuracy= 55.469\n",
      "\n",
      "Epoch 12, Validation Loss= 0.909, validation Accuracy= 49.800%\n",
      "Epoch 12, Average Training Loss= 0.886, Average Training Accuracy= 50.022%\n",
      "\n",
      "Iter 0, Loss= 0.889, Accuracy= 50.000\n",
      "Iter 20, Loss= 0.792, Accuracy= 53.906\n",
      "Iter 40, Loss= 0.948, Accuracy= 40.625\n",
      "Iter 60, Loss= 0.846, Accuracy= 56.250\n",
      "\n",
      "Epoch 13, Validation Loss= 0.916, validation Accuracy= 48.000%\n",
      "Epoch 13, Average Training Loss= 0.878, Average Training Accuracy= 50.714%\n",
      "\n",
      "Iter 0, Loss= 0.881, Accuracy= 48.438\n",
      "Iter 20, Loss= 0.871, Accuracy= 47.656\n",
      "Iter 40, Loss= 0.906, Accuracy= 46.875\n",
      "Iter 60, Loss= 0.858, Accuracy= 52.344\n",
      "\n",
      "Epoch 14, Validation Loss= 0.911, validation Accuracy= 50.100%\n",
      "Epoch 14, Average Training Loss= 0.870, Average Training Accuracy= 50.893%\n",
      "\n",
      "Iter 0, Loss= 0.822, Accuracy= 48.438\n",
      "Iter 20, Loss= 0.813, Accuracy= 58.594\n",
      "Iter 40, Loss= 0.931, Accuracy= 49.219\n",
      "Iter 60, Loss= 0.955, Accuracy= 44.531\n",
      "\n",
      "Epoch 15, Validation Loss= 0.910, validation Accuracy= 47.600%\n",
      "Epoch 15, Average Training Loss= 0.863, Average Training Accuracy= 51.585%\n",
      "\n",
      "Iter 0, Loss= 0.853, Accuracy= 54.688\n",
      "Iter 20, Loss= 0.829, Accuracy= 51.562\n",
      "Iter 40, Loss= 0.825, Accuracy= 60.156\n",
      "Iter 60, Loss= 0.891, Accuracy= 46.875\n",
      "\n",
      "Epoch 16, Validation Loss= 0.911, validation Accuracy= 49.900%\n",
      "Epoch 16, Average Training Loss= 0.859, Average Training Accuracy= 52.667%\n",
      "\n",
      "Iter 0, Loss= 0.876, Accuracy= 46.875\n",
      "Iter 20, Loss= 0.934, Accuracy= 43.750\n",
      "Iter 40, Loss= 0.792, Accuracy= 56.250\n",
      "Iter 60, Loss= 0.862, Accuracy= 50.000\n",
      "\n",
      "Epoch 17, Validation Loss= 0.922, validation Accuracy= 48.800%\n",
      "Epoch 17, Average Training Loss= 0.849, Average Training Accuracy= 52.768%\n",
      "\n",
      "Iter 0, Loss= 0.824, Accuracy= 57.812\n",
      "Iter 20, Loss= 0.842, Accuracy= 53.125\n",
      "Iter 40, Loss= 0.883, Accuracy= 53.125\n",
      "Iter 60, Loss= 0.946, Accuracy= 48.438\n",
      "\n",
      "Epoch 18, Validation Loss= 0.933, validation Accuracy= 45.900%\n",
      "Epoch 18, Average Training Loss= 0.838, Average Training Accuracy= 53.672%\n",
      "\n",
      "Iter 0, Loss= 0.850, Accuracy= 53.125\n",
      "Iter 20, Loss= 0.807, Accuracy= 56.250\n",
      "Iter 40, Loss= 0.936, Accuracy= 45.312\n",
      "Iter 60, Loss= 0.820, Accuracy= 57.812\n",
      "\n",
      "Epoch 19, Validation Loss= 0.967, validation Accuracy= 48.900%\n",
      "Epoch 19, Average Training Loss= 0.822, Average Training Accuracy= 54.319%\n",
      "\n",
      "Iter 0, Loss= 0.856, Accuracy= 53.125\n",
      "Iter 20, Loss= 0.849, Accuracy= 51.562\n",
      "Iter 40, Loss= 0.805, Accuracy= 62.500\n",
      "Iter 60, Loss= 0.812, Accuracy= 54.688\n",
      "\n",
      "Epoch 20, Validation Loss= 0.973, validation Accuracy= 49.000%\n",
      "Epoch 20, Average Training Loss= 0.820, Average Training Accuracy= 54.453%\n",
      "\n",
      "Iter 0, Loss= 0.720, Accuracy= 64.844\n",
      "Iter 20, Loss= 0.774, Accuracy= 54.688\n",
      "Iter 40, Loss= 0.751, Accuracy= 66.406\n",
      "Iter 60, Loss= 0.797, Accuracy= 58.594\n",
      "\n",
      "Epoch 21, Validation Loss= 0.984, validation Accuracy= 45.000%\n",
      "Epoch 21, Average Training Loss= 0.806, Average Training Accuracy= 55.882%\n",
      "\n",
      "Iter 0, Loss= 0.785, Accuracy= 60.156\n",
      "Iter 20, Loss= 0.714, Accuracy= 60.938\n",
      "Iter 40, Loss= 0.798, Accuracy= 56.250\n",
      "Iter 60, Loss= 0.807, Accuracy= 59.375\n",
      "\n",
      "Epoch 22, Validation Loss= 1.004, validation Accuracy= 44.200%\n",
      "Epoch 22, Average Training Loss= 0.778, Average Training Accuracy= 57.846%\n",
      "\n",
      "Iter 0, Loss= 0.744, Accuracy= 60.156\n",
      "Iter 20, Loss= 0.759, Accuracy= 60.156\n",
      "Iter 40, Loss= 0.724, Accuracy= 61.719\n",
      "Iter 60, Loss= 0.850, Accuracy= 57.031\n",
      "\n",
      "Epoch 23, Validation Loss= 1.011, validation Accuracy= 49.400%\n",
      "Epoch 23, Average Training Loss= 0.761, Average Training Accuracy= 59.554%\n",
      "\n",
      "Iter 0, Loss= 0.677, Accuracy= 64.844\n",
      "Iter 20, Loss= 0.862, Accuracy= 53.125\n",
      "Iter 40, Loss= 0.738, Accuracy= 57.031\n",
      "Iter 60, Loss= 0.776, Accuracy= 61.719\n",
      "\n",
      "Epoch 24, Validation Loss= 1.027, validation Accuracy= 46.500%\n",
      "Epoch 24, Average Training Loss= 0.751, Average Training Accuracy= 60.011%\n",
      "\n",
      "Iter 0, Loss= 0.753, Accuracy= 61.719\n",
      "Iter 20, Loss= 0.605, Accuracy= 75.000\n",
      "Iter 40, Loss= 0.677, Accuracy= 64.062\n",
      "Iter 60, Loss= 0.709, Accuracy= 67.188\n",
      "\n",
      "Epoch 25, Validation Loss= 0.999, validation Accuracy= 45.200%\n",
      "Epoch 25, Average Training Loss= 0.729, Average Training Accuracy= 61.652%\n",
      "\n",
      "Iter 0, Loss= 0.691, Accuracy= 64.844\n",
      "Iter 20, Loss= 0.667, Accuracy= 67.188\n",
      "Iter 40, Loss= 0.704, Accuracy= 67.188\n",
      "Iter 60, Loss= 0.683, Accuracy= 60.156\n",
      "\n",
      "Epoch 26, Validation Loss= 1.103, validation Accuracy= 45.500%\n",
      "Epoch 26, Average Training Loss= 0.693, Average Training Accuracy= 64.062%\n",
      "\n",
      "Iter 0, Loss= 0.651, Accuracy= 64.062\n",
      "Iter 20, Loss= 0.666, Accuracy= 67.969\n",
      "Iter 40, Loss= 0.722, Accuracy= 64.062\n",
      "Iter 60, Loss= 0.731, Accuracy= 65.625\n",
      "\n",
      "Epoch 27, Validation Loss= 1.058, validation Accuracy= 45.900%\n",
      "Epoch 27, Average Training Loss= 0.676, Average Training Accuracy= 64.676%\n",
      "\n",
      "Iter 0, Loss= 0.630, Accuracy= 74.219\n",
      "Iter 20, Loss= 0.668, Accuracy= 67.969\n",
      "Iter 40, Loss= 0.622, Accuracy= 64.062\n",
      "Iter 60, Loss= 0.674, Accuracy= 64.062\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 28, Validation Loss= 1.149, validation Accuracy= 46.000%\n",
      "Epoch 28, Average Training Loss= 0.657, Average Training Accuracy= 66.696%\n",
      "\n",
      "Iter 0, Loss= 0.720, Accuracy= 71.094\n",
      "Iter 20, Loss= 0.573, Accuracy= 72.656\n",
      "Iter 40, Loss= 0.679, Accuracy= 66.406\n",
      "Iter 60, Loss= 0.607, Accuracy= 70.312\n",
      "\n",
      "Epoch 29, Validation Loss= 1.186, validation Accuracy= 47.400%\n",
      "Epoch 29, Average Training Loss= 0.615, Average Training Accuracy= 69.431%\n",
      "\n",
      "Iter 0, Loss= 0.601, Accuracy= 76.562\n",
      "Iter 20, Loss= 0.523, Accuracy= 76.562\n",
      "Iter 40, Loss= 0.605, Accuracy= 67.969\n",
      "Iter 60, Loss= 0.640, Accuracy= 64.062\n",
      "\n",
      "Epoch 30, Validation Loss= 1.199, validation Accuracy= 47.900%\n",
      "Epoch 30, Average Training Loss= 0.585, Average Training Accuracy= 71.551%\n",
      "\n",
      "Iter 0, Loss= 0.556, Accuracy= 68.750\n",
      "Iter 20, Loss= 0.539, Accuracy= 76.562\n",
      "Iter 40, Loss= 0.515, Accuracy= 80.469\n",
      "Iter 60, Loss= 0.578, Accuracy= 73.438\n",
      "\n",
      "Epoch 31, Validation Loss= 1.266, validation Accuracy= 47.500%\n",
      "Epoch 31, Average Training Loss= 0.564, Average Training Accuracy= 72.344%\n",
      "\n",
      "Iter 0, Loss= 0.473, Accuracy= 76.562\n",
      "Iter 20, Loss= 0.512, Accuracy= 75.781\n",
      "Iter 40, Loss= 0.601, Accuracy= 73.438\n",
      "Iter 60, Loss= 0.446, Accuracy= 80.469\n",
      "\n",
      "Epoch 32, Validation Loss= 1.332, validation Accuracy= 49.900%\n",
      "Epoch 32, Average Training Loss= 0.536, Average Training Accuracy= 73.739%\n",
      "\n",
      "Early Stopping since best validation loss not decreasing for 20 epochs.\n",
      "\n",
      "Optimization Finished!\n",
      "\n",
      "Best Validation Accuracy: 50.700%\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess: # Start Tensorflow Session\n",
    "    \n",
    "    saver = tf.train.Saver() \n",
    "\n",
    "    sess.run(init) #initialize all variables\n",
    "    step = 1   \n",
    "    loss_list=[]\n",
    "    acc_list=[]\n",
    "    val_loss_list=[]\n",
    "    val_acc_list=[]\n",
    "    best_val_acc=0\n",
    "    best_val_loss=2**30\n",
    "    prev_val_acc=0\n",
    "    patience = 20\n",
    "    impatience = 0\n",
    "    display_step = 20\n",
    "    min_epoch = 20\n",
    "            \n",
    "    batch_size = 128\n",
    "    \n",
    "    while step <= epochs:\n",
    "        \n",
    "        total_loss=0\n",
    "        total_acc=0\n",
    "        total_val_loss = 0\n",
    "        total_val_acc = 0\n",
    "\n",
    "        batches_train_fact_stories,batches_train_questions,batches_train_answers = create_batches(train_fact_stories,train_questions,train_answers,batch_size)\n",
    "        \n",
    "        for i in xrange(len(batches_train_questions)):\n",
    "            \n",
    "            # Run optimization operation (backpropagation)\n",
    "            _,loss,acc = sess.run([optimizer,cost,accuracy],\n",
    "                                       feed_dict={tf_facts: batches_train_fact_stories[i], \n",
    "                                                  tf_questions: batches_train_questions[i], \n",
    "                                                  tf_answers: batches_train_answers[i],\n",
    "                                                  training: True})\n",
    "\n",
    "            total_loss += loss\n",
    "            total_acc += acc\n",
    "                \n",
    "            if i%display_step == 0:\n",
    "                print \"Iter \"+str(i)+\", Loss= \"+\\\n",
    "                      \"{:.3f}\".format(loss)+\", Accuracy= \"+\\\n",
    "                      \"{:.3f}\".format(acc*100)\n",
    "                        \n",
    "        avg_loss = total_loss/len(batches_train_questions) \n",
    "        avg_acc = total_acc/len(batches_train_questions)  \n",
    "        \n",
    "        loss_list.append(avg_loss) \n",
    "        acc_list.append(avg_acc) \n",
    "\n",
    "        val_batch_size = 100 #(should be able to divide total no. of validation samples without remainder)\n",
    "        batches_val_fact_stories,batches_val_questions,batches_val_answers = create_batches(val_fact_stories,val_questions,val_answers,val_batch_size)\n",
    "        \n",
    "        for i in xrange(len(batches_val_questions)):\n",
    "            val_loss, val_acc = sess.run([cost, accuracy], \n",
    "                                         feed_dict={tf_facts: batches_val_fact_stories[i], \n",
    "                                                    tf_questions: batches_val_questions[i], \n",
    "                                                    tf_answers: batches_val_answers[i],\n",
    "                                                    training: False})\n",
    "            total_val_loss += val_loss\n",
    "            total_val_acc += val_acc\n",
    "                      \n",
    "            \n",
    "        avg_val_loss = total_val_loss/len(batches_val_questions) \n",
    "        avg_val_acc = total_val_acc/len(batches_val_questions) \n",
    "             \n",
    "        val_loss_list.append(avg_val_loss) \n",
    "        val_acc_list.append(avg_val_acc) \n",
    "    \n",
    "\n",
    "        print \"\\nEpoch \" + str(step) + \", Validation Loss= \" + \\\n",
    "                \"{:.3f}\".format(avg_val_loss) + \", validation Accuracy= \" + \\\n",
    "                \"{:.3f}%\".format(avg_val_acc*100)+\"\"\n",
    "        print \"Epoch \" + str(step) + \", Average Training Loss= \" + \\\n",
    "              \"{:.3f}\".format(avg_loss) + \", Average Training Accuracy= \" + \\\n",
    "              \"{:.3f}%\".format(avg_acc*100)+\"\"\n",
    "        \n",
    "        impatience += 1\n",
    "        \n",
    "        if avg_val_acc >= best_val_acc:\n",
    "            best_val_acc = avg_val_acc\n",
    "            saver.save(sess, 'DMN_Model_Backup/model.ckpt') \n",
    "            print \"Checkpoint created!\"\n",
    "\n",
    "\n",
    "    \n",
    "        if avg_val_loss <= best_val_loss: \n",
    "            impatience=0\n",
    "            best_val_loss = avg_val_loss\n",
    "\n",
    "\n",
    "        \n",
    "        if impatience > patience and step>min_epoch:\n",
    "            print \"\\nEarly Stopping since best validation loss not decreasing for \"+str(patience)+\" epochs.\"\n",
    "            break\n",
    "            \n",
    "        print \"\"\n",
    "        step += 1\n",
    "        \n",
    "    \n",
    "        \n",
    "    print \"\\nOptimization Finished!\\n\"\n",
    "    \n",
    "    print \"Best Validation Accuracy: %.3f%%\"%((best_val_acc*100))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving logs about change of training and validation loss and accuracy over epochs in another file.\n",
    "\n",
    "import h5py\n",
    "\n",
    "file = h5py.File('Training_logs_DMN_plus.h5','w')\n",
    "file.create_dataset('val_acc', data=np.array(val_acc_list))\n",
    "file.create_dataset('val_loss', data=np.array(val_loss_list))\n",
    "file.create_dataset('acc', data=np.array(acc_list))\n",
    "file.create_dataset('loss', data=np.array(loss_list))\n",
    "\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEPCAYAAAC5sYRSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VFX6wPHvCYrUhN4xIEhREQF1pUlABUQUlN5EZcEf\niqy6rrKuLmBZsS6iIrri0ouLgiCg1FCUEgSkN8FA6C2ElpBk3t8fZxLSJpkJk0zJ+3me+2Tm3jP3\nvjOZeefMueeeY0QEpZRSwSXE1wEopZTyPk3uSikVhDS5K6VUENLkrpRSQUiTu1JKBSFN7kopFYTc\nSu7GmPbGmF3GmD3GmFey2F7dGLPMGLPRGLPZGPOg90NVSinlLpNTP3djTAiwB7gPOAJEAT1FZFea\nMl8AG0XkC2NMfWCBiNTMu7CVUkplx52a+93AXhGJFpFEYAbQKUMZBxDqvF0KOOy9EJVSSnnqOjfK\nVAUOpbkfg034aY0EFhljhgLFgPu9E55SSqnc8NYJ1V7Af0WkOvAQMMVL+1VKKZUL7tTcDwM3prlf\njczNLgOAdgAistYYU8QYU05ETqUtZIzRgWyUUioXRMR4Ut6dmnsUUNsYE26MKQz0BOZmKBONsynG\neUL1hoyJPU2AAbsMHz7c5zFo/L6Po6DFrvH7fsmNHJO7iCQDQ4BFwHZghojsNMaMNMZ0dBZ7CRho\njNkMTAX65yoapZRSXuFOswwi8iNQN8O64Wlu7wRaeDc0pZRSuaVXqHogIiLC1yFcE43fdwI5dtD4\nA1GOFzF59WDGSH4eTymlgoExBsmDE6pKKaUCjCZ3pZQKQprclVIqCGlyV0qpIKTJXSmlgpAmd6WU\nCkKa3JVSKghpcldKqSCkyV0ppYKQJnellApCmtyVUioIaXJXSqkgpMldKaWCkCZ3pZQKQprclVIq\nCGlyV0qpIKTJXSml3CECc+bAJ5/4OhK3aHJXSqmc/PwztGgB//wn1Knj62jcosldKaVc2bkTOneG\nXr1g0CDYtAnatfN1VG7R5K6UUhkdOWKT+b33QvPmsGcP9O8PhQr5OjK3aXJXSqkUcXHw2mvQoAGU\nKmWT+t/+BkWK+Doyj13n6wCUUsrnYmJg5kx47z148EHb/HLjjb6O6ppocldKFTwisHUrfP+9XQ4c\ngIcegsWL4fbbfR2dVxgRyb+DGSP5eTyllEqVlASrVl1N6ACdOtmlRQu4/nrfxpcNYwwiYjx5jNbc\nlVLBbeNG+OgjWLgQbrrJJvPvv7ft6sajfBlQtOaulApe06bBX/5iT5J26QLVqvk6olzJs5q7MaY9\nMBrbu2a8iLybYftHQGtAgOJAeREp40kgSinlNQ6HTejTp8OyZbaWXsDkWHM3xoQAe4D7gCNAFNBT\nRHa5KD8EuENE/pzFNq25K6Xy1vnz0LcvxMbCrFlQvryvI7pmuam5u9PP/W5gr4hEi0giMAPolE35\nXsB0T4JQSimvOHAAmjWDihVtz5cgSOy55U5yrwocSnM/xrkuE2PMjUANYNk1R6aUUp5YsQKaNrVX\nln7xBRQu7OuIfMrbvWV6ArO07UUpla++/BJefx2mToX77/d1NH7BneR+GEh7qVY157qs9ASeyW5n\nI0aMSL0dERFBRESEGyEopVQWEhPhxRdhyRJYvRpuvtnXEXlFZGQkkZGR17QPd06oFgJ2Y0+oHgXW\nA71EZGeGcvWABSJyUzb70kq9Uso7LlyARx+F666DGTMgLMzXEeWZPDmhKiLJwBBgEbAdmCEiO40x\nI40xHdMU7YE92aqUUnkrPt4m9urV4Ycfgjqx55ZexKSUCixJSdCtmx0uYPr0gBqGN7d0+AGlVHBz\nOOCpp2zNfebMApHYc0uTu1IqMIjA0KHwxx/w448FvqtjTjS5K6UCw2uvwZo1djiBYsV8HY3f0+Su\nlPJ/770H330HK1fqyVM3aXJXSvm3L76AcePsWOwFeDgBT2lyV0r5r+nT4c037dACVbMc9US5oMld\nKeWf5s2DF16wV5/WquXraAKOJnellP9ZsgQGDID58+G223wdTUByZ1RIpZTKHyIwejT06WPHYr/r\nLl9HFLC05q6U8g8XL8Kf/wy7d8O6dVCjhq8jCmhac1dK+d7evXDPPVCkCPz8syZ2L9DkrpTyrXnz\noHlzePZZ+PprKFrU1xEFBW2WUUr5RnIyjBgBEybA3Lm25q68RpO7Uir/nTkDvXtDQgJs2GDnPFVe\npc0ySqn8tWkT3Hmn7eK4eLEm9jyiNXelVN5KSICoKDt8wKpVsH49jB0L3bv7OrKgppN1KKW86/x5\n+OWXq8n811+hXj1o2dIu994L5cr5OsqAkpvJOjS5K6WunQh88IGdy3T3bmjS5Goib9oUSpb0dYQB\nTWdiUkrlv+RkeOYZ25Y+ZoxtT7/hBl9HVeBpcldK5V5iIjz+OBw/DkuXag3dj2hyV0rlzuXLdqLq\nkBBYsMBeXar8hnaFVEp5Li4O2reHUqXg2281sfshTe5KKc+cOgVt2sCtt8KkSXD99b6OSGVBk7tS\nyn2HD9seMG3bwmef2SYZ5Zf0P6OUcs/vv9vujf37w7/+Bcajnnkqn2lyV0rlbOtWW2N/5RW7KL+n\nyV0plb2FC+GBB+DDD+Hpp30djXKTdoVUSmXtxAk7QfWaNTB9OrRu7euIlAe05q6USk8EJk6EBg2g\nShXbJKOJPeC4VXM3xrQHRmO/DMaLyLtZlOkODAccwG8i0tebgSql8sH+/bbp5fRpe2FSkya+jkjl\nUo41d2NMCPAp0A64FehljKmXoUxt4BWgqYg0AJ7Pg1iVUnklKckO/HX33bab4/r1mtgDnDs197uB\nvSISDWCMmQF0AnalKTMQ+ExE4gBE5JS3A1VK5ZGNG2HgQChTBtatg1q1fB2R8gJ3kntV4FCa+zHY\nhJ9WHQBjzGrsr4GRIvKTVyJUSnlXYiLs2QNbtsDKlfDdd/Dee3YAMO27HjS81VvmOqA2cC9wI7DS\nGHNbSk0+rREjRqTejoiIICIiwkshKKXSEYGjR20S37r16t89e6B6dbj9dmjY0K6rUMHX0ao0IiMj\niYyMvKZ95DhZhzHmHmCEiLR33h8GSNqTqsaYz4G1IjLReX8J8IqI/JphXzpZh1J57dIl+Oc/YcIE\nWxNv2ND2fLn9dvv3llugWDFfR6k8kFeTdUQBtY0x4cBRoCfQK0OZOc51E40x5YCbgf2eBKKU8oLV\nq+Gpp+yEGZs2QbVq2tRSQOWY3EUk2RgzBFjE1a6QO40xI4EoEflBRH4yxrQ1xmwHkoCXRORs3oau\nlEp16RK8+ip8842dfLpzZ19HpHxM51BVKtCtXAkDBsCf/gQffwxly/o6IuVlOoeqUgXJxYvw97/b\nyTLGjoVOnXwdkfIjOvyAUoFoxQp7gjQ21vZ20cSuMtCau1KB5PJlePll2zd93Dh4+GFfR6T8lCZ3\npQLF779D165w882wbRuULu3riJQf02YZpQLB7NnQtKk9cTpzpiZ2lSOtuSvlzxITYdgwe9L0hx/s\nwF5KuUGTu1L+6vBh6NEDwsLg11+1i6PyiDbLKOWPliyxV5l26ADz5mliVx7TmrtS/sThgLfesj1h\npk3TGZBUrmlyV8pfnDwJ/frZ7o4bNtgp7pTKJW2WUcrXEhLgo4/saI2NGsHSpZrY1TXTmrtSviIC\n//uf7Q1z6632qtNbbvF1VCpIaHJXyhd+/hleesnW2r/6Ctq08XVEKshoclcqP+3bZ2vq69fD229D\nnz4Qoq2jyvv0XaVUfjh9Gl54Ae65Bxo3hl277MlTTewqj2jNXancio21Mx/FxEBcHJw/b5esbsfE\nQK9esH07VKzo68hVAaCTdSjlrnPnYNUqWL4cIiPtRNP33AM33QShoVCypF2yul2pkk5CrXItN5N1\naHJXypW4OJvMIyPtsmuXne0oIsJeXHTXXVC4sI+DVAWBJnelrlVcnB2Bcdo0+OUXm8Bbt7YJ/e67\n4YYbfB2hKoA0uSuVG/HxsGCBTeiLF9tk3rs3dOwIxYr5OjqlNLkr5bbkZNt2Pm0azJkDd9xhE3qX\nLjpWuvI7mtxVwRYXB7/9BpcuZV4uXrx6OzYWFi6EqlVtQu/Rw95Wyk9pclcF044d8NlnMH061Ktn\ne6cUK+Z6KV7cNr3UrevryJVyS26Su/ZzV4EpKQnmzoVPP4WdO2HQINi6VWvgSjlpcleB5cQJOxbL\nuHFw440wZAg89ph2SVQqA03uyv85HLB2rU3o8+ZB16621n7HHb6OTCm/pW3uyj/98YftlrhkiR3f\nvEIFGDAAnnwSypTxdXRK5Ss9oaoCV2ys7Zq4eLFdzp2D+++HBx6wf6tX93WESvlMniV3Y0x7YDR2\nFMnxIvJuhu39gfeBGOeqT0Xk6yz2o8m9oLl8Gc6cubqcPZv+/pkzsHmzHVCrWTObzB94ABo00BET\nlXLKk+RujAkB9gD3AUeAKKCniOxKU6Y/0EREhuawr6BO7rt22W7TQ4bAE08UwNzkcNh+5im17zVr\nIDERypa1FwaVKXN1SXu/bl1o3hyKFPH1M1DKL+VVV8i7gb0iEu08yAygE7ArQzmPDhxszp2DTp2g\ne3fbmePzz2HMGGja1NeR5bGDB9O3jZcubZtRnn0WvvkGSpUCU6DfGkr5hDvJvSpwKM39GGzCz+gx\nY0xLbC3/RRGJyaJMUHI47IQ6998Pb74JI0fC1Km2U8d998GoUZ7Nd5wy1MnSpfDXv9oRZX0qKQlO\nnrTdEE+cgGPHYN06m9TPnLFP8oEH7BMND3d7tyIwfjwULWpfP6UCQWIiXH+9r6PImbe6Qs4FpolI\nojFmEDAR24yTyYgRI1JvR0REEBER4aUQfGf4cHvl+7//be+HhNhJdjp3hn/9C26/Hf72N3j+edeD\nCiYlpR/qpFEjaNgQWra0vf8aN87jJ3H5Mvz4ow3i+HG7pCTzc+egTBmkYkXiilTg8JUKFG/WkPDp\n0213xFy0P125YpuvfvnFvnZxcTB4cO7DdzjgH/+w4Q8caJvHihbN/f6UykjEfp737IGJE/P2WJGR\nkURGRl7bTkQk2wW4B/gxzf1hwCvZlA8BYl1sk2Dz7bci1auLHDvmuszevSIPPyxSu7bI3LkiDodd\n73CIrFkj8txzIhUritx5p8hHH4nExFx97HffiZQvL/LTT2l26HCI7N4tMmGCyP/9n0jv3iJjx4rs\n2HF15+64fFlk9myRXr1EwsJEWrcWef99kenTRZYuFdm2TQ5tPCFffZEk3buLlCkjcsst9pCVKon8\n/e8iCQkevVwiInL8uEiLFiKdOonExYn8/rtIjRoi//635/sSsTH07i3SrJnInDkiDz0kUq6cyCuv\niERH526fgc7hEElO9nUUwcPhEPnrX0UaNBA5ciT/j+/MnTnm67SLO8m9ELAPCAcKA5uB+hnKVEpz\n+1HgFxf7yoeXIf9s3WqTSFSUe+UXLhSpW1ekfXubGGvWtPdHjhTZs8f143758Zx0KbVENnV5U6RD\nB5tlb7xRpHt3mxHHjxfp399myAoVRLp2Ffn0Uxtgxk94SkLv3ftqQv/889Rvp7g4+wX03HM2tvLl\nbe7/739FDh26uptjx0Q6dhRp3Nh+p7hr40Yb+uuvpw/tjz9EatUSGTXK/X2JiJw7J3L//SKdO4tc\nunR1/d69Is8/b1+qxx4TWb7cs++9QPfGG/bL7uJFX0cS+JKSRJ56SuSee0ROn/ZNDLlJ7p50hfyY\nq10hRxljRgJRIvKDMeZfwCNAInAGGCwie7LYj7hzvEBw9qydx+Gf/4THH3f/cVeu2JOtR45Az562\nVSPT+cboaDsD0KpVtsfJ779zqW4jJu9rSsVO99BpVFNMVReN+NHRsGLF1SU2Fu6913Yz3LwZ5s+3\nB+3e3V6275zPMznZ/uR8/307J8UDD0DbtrZpyFWriwh8+SW89pptmnr22ezPnc6caZtixo6Fbt0y\nbz982Dbf9+kDr7+e82t57Bg8+KCd6e7TT6FQocxlLlyASZPgk09sO+lzz9n9e2uY9hMn7M/0rAai\nTLtUrAgvvZQ/55bPnoWbb7Yn80NC4Ntv4Tq9Fj1XEhLs+yUuDr77DkqU8E0cehFTXnI47N+QEJKT\n7TwOdevC6NHXuF8RO/DVypVXE3pCgm1sb9nSJuWGDaFwYWJibDJr0wY++ijrZJZJTIxN8mvWwC23\n2IReqVK6IkeP2jewiD0R7MnJX7DJrV8/21Hm668zP97hsMl66tSrQ6e7cuyYTfCPPmpPTrtKhnv2\nQPv28NRTtq09p6QpYjv0fPKJfSlmzYJWrTx7nhkdPGgTaM2adqDJtANPFi2a/v6ECfD3v+fPieMR\nI+x3/BdfwEMPQe3a9gtVOy155uJF+3EpUcKeC/PlJFya3HMhOtrWPLt2hQ4dsjgLvns3/Pe/tvp3\n/DiEhnLWEcZpRylualyKkFJhtrtfmPNvaKjdSaFCdrnuusy3r7vuau189Wr72JRk3rKlrXa5+CTG\nxtoulxUqwOTJ1941/KefbJ/8wYNtknTrCyMLiYnw1lt2+JexY+2cF2BrPH372rhnzXJvjuiTJ69e\nmPr++5lfirVrbfJ/+22b3D21ZIlNssuWwa23ev54sM+reXM7GsKLL+ZcfsMGWyHYuhXKl8/dMd1x\n7hzUqmW/wG6+2cbZqpX9pfTqq3l33GBz9qz9YqxXz/469fUvH03uudCpk61lHToE+/fbRDeg+3lq\nb/qfrYbu22fbXZ58EurUYfbEOD58PZZ5k2MpHXLOZq3YWPupio21n6akJLskJ9slq9sVK15N5h4O\nUxsfb0M6ccLWhEuV8vx5JybaL7Vp02DKlGuvxaZYu9Ym8xYtYOhQW6Nv2dL2+fdk4MYzZ2yzULNm\n8PHHVxP8Dz/Yf8WECfbDl1tTp9pk98svno8SnJRkE3XNmp7ViF96yf4ymTLF83jd9eabsHevrYuk\nOHLEvo5vvOFZE2JBdewYtGtnf0F+8IF/XIyYm+TuUQP9tS742QnV77+3Jw3j40XE4ZADk1bK+luf\nkFgTJqvKdpJlz38vF2OvpJb/7Td7AnXjRt/FnCI5WWToUJHbbvO8R8iBA/bkUIcOIidOeD+28+dF\n/vxnkeuus514cis21sY5aJB9vv/5j+2ls26dd+IcNcr2foiNdf8xDoftLdS+vUhiomfHu3BB5Kab\nRBYs8Oxx7jp3zr4/d+3KvG3HDnuuPV2vqyAXHy+yYYPIl1+KjBkjsmiRyMGD2Z9YP3DA9mp74w3/\nOgFPXvSW8ebiT8n9wgWR8HCRpT8l2k/5zTeL1K8v8v77khB9VGbNEnnwQdvb4plnbG+LmjVFpk3z\ndeRXORwi770nUqKETVKDB4tMmWJ7nrh6Y6Z0rfzgg7zvKudJ0nQlLk6kZUvbTfSmm2wPUG9xOESe\nfVakTRv3u3R++KF9rc+dy90xFy+2vYXi4nL3+Oy8/bbtBOXKqlX2f+8PlRNvu3BB5OefRT75ROTJ\nJ0UaNhQpUsT+r/r3t1/IrVuLVK4sUry47eXVq5ftqTZzpq24bdwoUq2a/SLwN5rcPTBsmP3nyo8/\n2qS+dm2WGfHgQfstXru2yD/+kf9xuuPKFZH1620f+S5dbJ/5atVEeva0PSI3bbJd4oYMsV9Qa9f6\nOmLPXLhgX/ujR72/76QkkUcftUkxpy+72bNFqla99r7zTz5pu5p6U1ycTdw5dUudNUukShVbQw10\nW7bYxF2/vkjRorYCMGiQyLhx9vOQtmtsWrGx9tffpEkir75qu8reeqvtGTxpUr4+BbdpcnfT9u32\n5+vRoyLy0ks2ewcRh8P28/7vf23zSL16tomka1eRs2d9HZ3/uXRJpGlTkZdfdl0mKsqzaxqyc/q0\nrUH+8su17yvFqFEiPXq4V/bjj+17wld9tq/VwYMiTzxhm5nef9/Wuq9cyflxgUyTuxscDpFWrezP\nNxERueMO737K/NT58/7VhuhvTp0SqVMnzfsijehoW9udPdt7x/vmG1vjjI+/9n2dP28T3bZt7j/m\npZdEmjd3Xbv1R2fP2quOy5SxNW5vNPsFitwkdz84D5y/pkyB8+ed45icPAkHDtirkYJciRLazzk7\nZcvaoXXeecderJIiLs72yvnrX+1YQd7StavtqvjOO9e+r3HjbG8nT7p1vvuunf+kb197WYU/S0iw\n13XUqQOnT8OWLbYbbFiYryPzc55+G1zLgo9r7mfO2N4W69c7V8yYYQd9Ucppwwbb/LJ6te0N066d\nPRmXF796YmLssTypcWd08aI9x7Jli+ePjY+3TXXVqtkRKHIzTlBeSk62HQTCw+3HdPt2X0fkO+TV\n8APe4ut+7oMH29rr2LHOFQMH2hl/hmY7x4gqYH78Efr3h4gIW3OfNy/vLmIZN86OMLh6de4uIPv3\nv+1jv/029zGsW2evat2xw17I9sQTnl2T4AkROyTExYvZD9dw/rwdDvr66+2FbPfemzfxBAq9iCkb\n69fbC5Z27LCXyQN2oPQffrCX5SuVxsSJ8J//2HH1Q0Pz7jgOh/0S6drV8zrG5cv2atQFC7If0sFd\na9bYMYL27rUXuD3+eO7HLU9Ksi2eu3ZlXuLjoWTJ9MMzZFyKFrVXKXfpos2JoMndpeRkOxjW88/b\nKyYBezlqixZ2tCp99ygf2r3bDmXw668ezXXCmDF2CIU5c7wbz+rVNsn/8YcdE6hv38y/XETsBdkH\nD9qruw8etCNq7N1rE/iBA3aMoXr17BhM9epdXcqX14+cpzS5u/DJJ/Yk2bJlad5UX35px3aZPDnf\n41Eqo3/9y74dFyxwL/HFx9tae15O5LJihR31NGUE0xMnbBJPWUJC4MYbry7Vq9uTxPXq2b86Ja73\naHLPwtGjdiaklSuhfv00G7p3t90g+vfP13iUykpiItx5p/2F2bevrcln187/6ad20Ld58/I2LhE7\nu9WyZVCtWvpErr1V8o8m9yz07g01atiaUSqHww5PuHmzfccq5QdiYuxYdXPm2KaOhx6y3S/btk0/\n/nxCgh3Gd/Zs+4Wggp8m9wyWLIE//9meRE03OcOmTTbr79yZb7Eo5YnoaJg71yb6qCho3dom+o4d\n7dDJP/xg511RBYMm9wxatLAnUbt2zbDh/fdto+Enn+RbLErl1pkzti1+zhxYvNg24SxfDn/6k68j\nU/lFk3sacXH2bP3Jk7ZbVTrt2tlO79685FCpfBAfb3ujeKProwocuUnuQTv8wIoVdm7NTIk9Pt7O\n0BAR4YuwlLomRYpoYlfuCdrkvmSJnaYtkzVr7CAcuZm+SCmlAkTBS+5Ll7rYoJRSwSMok/uRI3Ye\nxEaNstjoMusrpVTwCMrkvnSp7TqWaSCm2FjYvh2aNvVJXEoplV+CMrm7rJxHRtpp4G+4Ib9DUkqp\nfBV0yV0km+SuTTJKqQIi6JL7rl12mNJatbLYqCdTlVIFRNAl96VL4b77shhZLyYGTp2Chg19EpdS\nSuWnoEvu2XaBbN3ajlOqlFJBzq1MZ4xpb4zZZYzZY4x5JZtyXYwxDmNMHo0wnb2kJHvOtE2bLDZq\ne7tSqgDJMbkbY0KAT4F2wK1AL2NMvSzKlQCGAmu9HaS7NmywM9lUrJhhQ7ZnWZVSKvi4U3O/G9gr\nItEikgjMADplUe5NYBSQ4MX4POIyf+/caQeZuemmfI9JKaV8wZ3kXhU4lOZ+jHNdKmNMI6CaiCz0\nYmwe0y6QSillZTORl3uMMQb4CEg7X53LoSlHjBiRejsiIoIIL43OePGibZZp2TKLjUuWQJ8+XjmO\nUkrltcjISCIjI69pHzmO526MuQcYISLtnfeHASIi7zrvhwL7gAvYpF4JOA08IiIbM+wrz8Zz//FH\neOcdO9RvOomJUK4c7Ntnp11XSqkAk5vx3N2puUcBtY0x4cBRoCfQK2WjiMQBFdIEsRx4UUQ2eRLI\ntUrp355JVJRta9fErpQqQHJscxeRZGAIsAjYDswQkZ3GmJHGmI5ZPYRsmmXyig7xq5RSVwXFNHsn\nT9rZ4E+dskMPpNOqFfz979C+vdePq5RS+aHATrO3bJnN4ZkS+4UL8OuvLs6yKqVU8AqK5O6ySWbV\nKrjzTihePN9jUkopXwr45C4Cixdr/3allEor4JP7/v1w5QrUr5/FxpUrwUv96JVSKpAEfHJPqZxn\nGuI3IQF27IDGPhnDTCmlfCrgk7vLno5bttguNMWK5XtMSinlawGd3B0O21Mmy4uXNmyAu+7K95iU\nUsofBHRy37zZXnhatWoWG6OibE8ZpZQqgAI6uWfbGSYqSmvuSqkCKziT+8WLthtNgwb5HpNSSvmD\ngE3u8fGwZo2Lno4bN8Jtt0HhwvkdllJK+YWATe6//GLzd1hYFhs3bND2dqVUgRawyV3b25VSyrWA\nTe7ZjuSryV0pVcAF5JC/Z89CeLgd6veGG7LYeOONEBsLhQpd87GUUsrXCsyQv0uWQPPmWSR2sEP8\nNmqkiV0pVaAFZHL/5hvo0sXFRm2SUUqpwEvuFy7AokXw2GMuCmhyV0qpwEvuP/xgm2TKlHFRQJO7\nUkoFXnKfORO6d3ex8dgxe3XqTTfla0xKKeVvAiq5x8XZUSA7d3ZRIOXipUyDuyulVMESUMl97ly4\n914oVcpFAW2SUUopAK7zdQCemDkTevTIpkBUFAwalG/xqOBQo0YNoqOjfR2GUoSHh/PHH394ZV8B\ncxFTbKy9cOnQIQgNzaKACFSsaAcNq1bt2gJVBYrzAhFfh6GUy/diUF/ENGcOtGnjIrEDHDxoL1zK\ncuYOpZQqWAImuX/zjRtNMnfdpSdTlVKKAEnup0/Dzz9Dx47ZFNKTqUoplSogkvvs2dC2LZQokU0h\nHcNdKaVSuZXcjTHtjTG7jDF7jDGvZLH9aWPMFmPMJmPMSmNMPW8GmWOTjMNhBwzTmrtS6URHRxMS\nEoLD4QCgQ4cOTJ482a2ynnrnnXcYpL3V/IeIZLtgvwD2AeHA9cBmoF6GMiXS3H4YWOhiX+KpEydE\nwsJELl7MptCuXSI1ani8b6VERHLzvswv7du3l+HDh2daP2fOHKlUqZIkJydn+/g//vhDQkJCcizn\nadnIyEijDEngAAAXuElEQVSpVq1ajuW8afny5WKMkffeey9fj5ufXL0XnetzzNdpF3dq7ncDe0Uk\nWkQSgRlApwxfEBfS3C0B5O6rPwvffQcPPgjFimVTSNvbVZDq378/U6ZMybR+ypQp9OvXj5AQ37Ss\niggmnzsvTJo0ibJlyzJp0qR8PS5AcnJyvh/zWrnzzqgKHEpzP8a5Lh1jzDPGmH3AKGCod8LLYSyZ\nFBs2aHJXQalz586cPn2a1atXp66LjY3lhx9+4PHHHwdgwYIFNG7cmLCwMMLDwxk5cqTL/bVu3Zqv\nv/4aAIfDwUsvvUT58uWpXbs28+fPT1d2woQJ3HLLLYSGhlK7dm2+/PJLAC5dukSHDh04cuQIJUuW\nJDQ0lGPHjjFy5Ej69euX+vi5c+dy2223UaZMGdq0acOuXbtSt9WsWZMPP/yQhg0bUrp0aXr16sWV\nK1dcxn3p0iVmzZrFZ599xt69e9m4cWO67atXr6Z58+aULl2a8PDw1C+A+Ph4/vrXv1KjRg1Kly7N\nvffeS0JCAitWrKB69erp9lGzZk2WLVsGwMiRI+nWrRv9+vWjVKlSTJw4kaioKJo1a0bp0qWpWrUq\nzz33HElJSamP3759O23btqVs2bJUrlyZUaNGcfz4cYoXL87Zs2dTy23cuJEKFSrk/RdGTlV7oAvw\nZZr7fYEx2ZTvCUxwsc2jnyhHj4qUKiVy+XIOBZs1E1m2zKN9K5XC0/dlfhs4cKAMHDgw9f64ceOk\nUaNGqfdXrFgh27ZtExGRrVu3SqVKleT7778XkcxNLRERETJ+/HgREfn888+lfv36cvjwYTl79qy0\nbt06XdkFCxbIgQMHRERk5cqVUqxYMdm0aZOI2GaZ6tWrp4tzxIgR0q9fPxER2b17txQvXlyWLl0q\nSUlJ8t5770nt2rUlMTFRRERq1Kghf/rTn+TYsWNy9uxZqV+/vnzxxRcuX4NJkyZJlSpVxOFwyMMP\nPyxDhw5N3RYdHS0lS5aUmTNnSlJSkpw5c0Z+++03ERF55plnpHXr1nL06FFxOByyZs0auXLlSpbx\n16hRQ5YuXZr6XAoXLixz584VEZH4+HjZuHGjrFu3ThwOh0RHR8stt9wiH3/8sYiInD9/XipXriz/\n/ve/JSEhQS5cuCDr168XEZGHHnpIxo0bl3qcF154IV38abl6L5KLZhl3kvs9wI9p7g8DXsmmvAFi\nXWyT4cOHpy7Lly/P8omk+PRTkb59sy0ikpgoUry4yLlzORRUKmvuJHd7CfS1Lbm1evVqKVWqlCQk\nJIiISPPmzWX06NEuyz///PPy4osvikj2yb1NmzbpEuqiRYuybXPv3LmzjBkzRkRyTu5vvvmm9OjR\nI3Wbw+GQqlWryooVK0TEJtJp06albn/55Zdl8ODBLp/T/fffn/qcpk+fLhUqVJCkpCQREXnnnXfk\nsccey/QYh8MhRYsWla1bt2ba5k5yb9Wqlct4RERGjx6detzp06dL48aNsyw3c+ZMad68uYiIJCcn\nS6VKlSQqKirLsinvxeXLl6fLlblJ7u6MLRMF1DbGhANHnTXzXmkLGGNqi8g+592OwB5XOxsxYoQb\nh7RmzoS//S2HQtu3Q/Xq2Vy6qtS18+XoBM2bN6d8+fLMmTOHO++8k6ioKGbPnp26ff369QwbNoxt\n27Zx5coVrly5Qrdu3XLc75EjR9I1TYSHh6fbvnDhQt544w327NmDw+Hg8uXL3H777W7FfOTIkXT7\nM8ZQvXp1Dh8+nLquYsWKqbeLFSvG0aNHs9xXTEwMy5cvZ9SoUQA88sgjDBo0iPnz5/PII49w6NAh\natWqlelxp06dIiEhgZtyOQR4xmabvXv38uKLL7JhwwYuX75MUlISTZo0AXAZA0CnTp0YPHgw0dHR\n7Ny5k1KlSnFnDt22IyIiiIiISL2fXVObKzm2uYtIMjAEWARsB2aIyE5jzEhjTMplRUOMMduMMRuB\n54H+HkeSweHDsG2b7d+eLW1vVwVAv379mDhxIlOmTKFdu3aUL18+dVvv3r3p3Lkzhw8fJjY2lqef\nftqtsXIqV67MoUNXT6elHTztypUrdO3alZdffpmTJ09y9uxZHnzwwdT95nQytUqVKpkGYzt06BDV\ncjHu06RJkxARHn74YSpXrkytWrVISEhg4sSJgE3C+/bty/S4cuXKUaRIEX7//fdM24oXL86lS5dS\n7ycnJ3Py5Ml0ZTI+x8GDB1O/fn1+//13YmNjefvtt1Nfj+rVq2d5HIAbbriB7t27M3ny5NQT4fnB\nrVPtIvKjiNQVkZtFZJRz3XAR+cF5+3kRuU1EGovIfSKy81oDmzULHnnExSTYaUVF6cVLKug9/vjj\nLFmyhK+++or+/dPXnS5cuEDp0qW5/vrrWb9+PdOmTUu33VWi7969O2PGjOHw4cOcPXuWd999N3Vb\nyi+AcuXKERISwsKFC1m0aFHq9ooVK3L69Gni4uJc7nv+/PksX76cpKQkPvjgA4oUKULTpk09fu6T\nJk1ixIgRbN68md9++43ffvuNWbNmMX/+fM6ePUufPn1YunQps2bNIjk5mTNnzvDbb79hjOHJJ5/k\nxRdf5OjRozgcDtauXUtiYiJ16tQhPj6ehQsXkpSUxFtvvZXtCV2A8+fPExoaSrFixdi1axeff/55\n6raOHTty7NgxxowZw5UrV7hw4QLr169P3d6vXz8mTJjAvHnz/Cu5+0KOFy6l0G6QqgAIDw+nWbNm\nXLp0iUceeSTdtrFjx/L6668TFhbGW2+9RY8MH5y0NdC0twcOHEi7du1o2LAhd955J13SzDpfokQJ\nxowZQ7du3ShTpgwzZsygU6erPaDr1q1Lr169uOmmmyhTpgzHjh1Ld8w6deowZcoUhgwZQvny5Zk/\nfz7z5s3juuuuyxRHdtatW8fBgwd55plnqFChQury8MMPc/PNNzN9+nSqV6/OggUL+OCDDyhTpgyN\nGjViy5YtAHzwwQc0aNCAu+66i7JlyzJs2DAcDgehoaGMHTuWAQMGUK1aNUqWLJnjr4oPPviAqVOn\nEhoaytNPP03Pnj3TvV6LFy9m7ty5VKpUiTp16hAZGZm6vVmzZoSEhNC4ceNMzT15xS+H/D10CBo1\ngqNH4frrsykYH28nUz19GooW9V6gqkDRIX9Vfrjvvvvo06cPTz31lMsy3hzy1y8n6/jmGzuVXraJ\nHWDLFqhbVxO7UsqvRUVFsWnTJubOnZtvx/TLZhmPmmS0vV0p5ceeeOIJ2rZty8cff0zx4sXz7bh+\nV3M/cMAurVu7UTgqCpo1y/OYlFIqtyZMmOCT4/pdzf1//4PHHoPr3OqBrydTlVIqK36X3HOcBDvF\nhQvwxx9w2215HZJSSgUcv2uWGT3azZaWjRuhQQM3zroqpVTB43fJvWVLNwtqk4xSSrnkd80ybtPk\nrpRSLgVuctcxZZTymMPhoGTJksTExHi1rPI/gZncz5yBEyegTh1fR6JUnkqZDCM0NJRChQpRrFix\n1HXTp0/3eH8hISGcP3/erQG8PCmbW1999RUhISHpRrlU3hFYyV0Efv4ZnnkG7r4bChXydURK5anz\n588TFxdHXFwc4eHhzJ8/P3Vdr169MpUPtOngfDl1Xm4nAg8UgZHcY2LgX/+yQw0MGGAHnskw8p1S\nwS5lEoa0Xn/9dXr27Env3r0JCwtj6tSprF27lqZNm6ZOB/eXv/wlNeknJycTEhLCwYMHATta4V/+\n8hc6dOhAaGgozZs3Tx2q15OyYMd/r1u3LqVLl2bo0KG0aNEi26T9+++/88svv/Dll1+yYMECTp8+\nnW77d999R6NGjQgLC6NOnTosWbIEgDNnzvDkk09SpUoVypYtmzp2/fjx42md5urHrOIfMmQIDz74\nICVLlmT16tXMmzcv9Rg1atTgrbfeShfDypUradq0KaVKlSI8PDz19a1aNf1Mo998802OY7TnO09n\n97iWBU+mo7l8WWT6dJF27URKlxYZNEhkzRoRh8P9fSjlBo/elz6UdqagFK+99prccMMNMn/+fBGx\n08Ft2LBB1q9fLw6HQw4cOCB169aVzz77TEREkpKSJCQkRKKjo0VEpG/fvlK+fHnZuHGjJCUlSY8e\nPVJnU/Kk7PHjx6VkyZIyb948SUpKko8++kgKFy4sEydOdPl8/vnPf6bOUFS/fv3UWZ5ERH7++Wcp\nVapU6mxtMTExsmfPHhERadu2rfTp00fOnTsnSUlJsmrVKhER+eqrr6R169ap+8gq/jJlysi6detE\nRCQhIUGWL18uO3bsEBGRLVu2SPny5VNfy/3790uJEiVk1qxZkpycLKdPn06dvq9evXqyZMmS1GM9\n/PDD8sknn2T7/3OHq/ciuZiJyb9q7iKwfr1tdqlaFcaPh8cftzX3L76Ae+6BfJ5xXSnAvu+udckj\nLVq0oEOHDoCdGKJJkybcddddGGOoUaMGAwcOZMWKFanlJUPtv2vXrjRq1IhChQrRp08fNm/e7HHZ\n+fPn06hRIzp27EihQoV44YUXKFu2bLZxT548mT59+gB2wpG0tfyvv/6aQYMGpc5GVLVqVW6++ebU\nWZnGjRuXeh6iRYsWLo+RMf5HH32Uu+++G4DChQsTERFB/fr1AWjQoAE9evRIfa2mTp1Khw4d6NKl\nCyEhIZQpUyZ1Jqp+/foxefJkwM74tGzZsnRDAPsD/0ruxsA770CVKrBpEyxeDL17Q7Fivo5MFXTe\nmEY1j2QcH3z37t107NiRypUrExYWxvDhwzl16pTLx1eqVCn1drFixbhw4YLHZTNO2QdkeyJ2xYoV\nHD58mO7duwPQq1cvfv31V3bs2AG4nrbu0KFDlCtXjhIlSrjcd3YyxrhmzRpat25NhQoVKFWqFOPH\nj099rbKbOq9fv37MnTuXhIQEZsyYQevWrSlXrlyuYsor/pXcAWbPhtdegxtv9HUkSgWEjBNfPP30\n0zRo0ID9+/dz7tw5Ro4cmefj1Wecsg9IN19qRhMnTsThcNCgQQMqV65MixYtCAkJSTd1XlbT1lWv\nXp1Tp05l+QWUceq8o0ePZnptMt7v1asX3bp1S52icMCAAemmzstq+r6UbU2aNGH27Nn5OnWeJ/wv\nuSulrsn58+cJCwujaNGi7Ny5ky+++CLPj9mxY0c2bdrE/PnzSU5OZvTo0S5/LVy+fJlvv/2Wr7/+\nOt3UeR999BFTpkxBRBgwYABfffUVK1asQEQ4fPgwe/bsoVq1atx///08++yznDt3jqSkJFatWgVA\nw4YN2bJlC9u3b+fy5cu88cYbOcaddorCtWvXMmPGjNRtffv25aeffmL27NkkJydz+vTp1BmewNbe\n33nnHXbv3p1ulip/ocldqQDh7tR0H374IRMmTCA0NJTBgwdnagt2Ne1eTsfMrmyFChWYOXMmL7zw\nAuXKlePAgQM0atSIG7KYBPm7774jNDSUPn36pJs6b+DAgcTHx7N48WKaNm3Kf/7zH5577jnCwsJo\n06ZN6sVUKV8AderUoVKlSnz66acA1K9fn1dffZVWrVpRv359WrVq5fK5pPj8888ZNmwYYWFhjBo1\nKt0UhTVq1GDevHmMGjWKMmXK0KRJE7Zt25a6vUuXLuzfv59u3bpl+Tx9zS+n2VMqP+k0e97ncDio\nUqUK3377Lc2bN/d1OHmmZs2aTJw4kXvvvdcr+/PmNHtac1dKecVPP/3EuXPnSEhI4I033qBw4cKp\nPVOC0cyZMylSpIjXEru3+d2okEqpwLR69Wp69+5NcnIyt956K3PmzOH6IB2Su2XLluzbt49pfnwx\npTbLqAJPm2WUv9BmGaWUUtnS5K6UUkFIk7tSSgUhPaGqCrzw8HC3+5ArlZfCw8O9ti+3TqgaY9oD\no7E1/fEi8m6G7S8AfwYSgZPAUyJyKIv96AlVpZTyUJ6cUDXGhACfAu2AW4Fexph6GYptBJqIyB3A\nt8D7ngQRKCIjI30dwjXR+H0nkGMHjT8QudPmfjewV0SiRSQRmAGkG0hBRFaISLzz7lqgKkEo0N8g\nGr/vBHLsoPEHIneSe1UgbRNLDNkn7wHAwmsJSiml1LXx6glVY0xfoAnQKqeySiml8k6OJ1SNMfcA\nI0SkvfP+MOyUTxlPqt4PfAzcKyKnM+/JnlD1StRKKVXAeHpC1Z3kXgjYDdwHHAXWA71EZGeaMo2A\n/wHtRCTzCPtKKaXyVY5t7iKSDAwBFgHbgRkistMYM9IY09FZ7D2gOPA/Y8wmY8ycPItYKaVUjvJ1\n4DCllFL5I9+GHzDGtDfG7DLG7DHGvJJfx/UWY8wfxpjfnL9M1vs6npwYY8YbY44bY7akWVfaGLPI\nGLPbGPOTMSbMlzG64iL24caYGGPMRufS3pcxZscYU80Ys8wYs90Ys9UYM9S5PlBe/4zxP+dcHxD/\nA2PMDcaYdc7P6lZjzHDn+hrGmLXOHDTdGON3V+hnE/t/jTH7nes3GmNuz3FnIpLnC/ZLZB8QDlwP\nbAbq5cexvfgc9gOlfR2HB/G2AO4AtqRZ9y7wsvP2K8AoX8fpQezDgRd9HZub8VcC7nDeLoE9Z1Uv\ngF5/V/EH0v+gmPNvIey1N38CZgLdnOs/B572dZwexP5f4DFP9pNfNfccL4QKAIYAGmhNRFYDZzOs\n7gRMdN6eCHTO16Dc5CJ2sP8Dvycix0Rks/P2BWAnUI3Aef2zij/l2pZA+R9cct68AdvlW4DW2Cvo\nwb7+j/ogtBxlEbvDed8vx3P39EIofyTAT8aYKGPMQF8Hk0sVROQ42A8wUMHH8XjqWWPMZmPMV/7a\npJGRMaYG9lfIWqBioL3+aeJf51wVEP8DY0yIMWYTcAxYDPwOxIpISqKMAar4Kr7sZIxdRKKcm95y\nvvYfGmNynOIqYGqifqC5iNwJdMC+wVv4OiAvCKSz6WOBWmLHLzoGfOTjeHJkjCkBzAL+4qwBZ3y9\n/fr1zyL+gPkfiIhDRBphfzHdjW1WCggZYzfG3AIME5H6wF1AWWyzXrbyK7kfBm5Mc7+ac13AEJGj\nzr8ngdnYN0ygOW6MqQhgjKkEnPBxPG4TkZPibIgE/oN9k/st58m6WcBkEfneuTpgXv+s4g+0/wGA\niMQBkUBToJRzIEQIgByUJvb2aX7xJWLb33PMP/mV3KOA2saYcGNMYaAnMDefjn3NjDHFnLUYjDHF\ngbbANt9G5RZD+na6ucATztv9ge8zPsCPpIvdmQxTPIb/v/5fAztE5OM06wLp9c8Uf6D8D4wx5VKa\njIwxRYEHgB3AcqCbs5hfvv4uYt+V8tobO/FAZ9x47fOtn7uz29THXB0TflS+HNgLjDE1sbV1wZ7g\nmOrv8RtjpgER2J9wx7E9HeZgrySuDkQD3UUk1lcxuuIi9tbYtl8H8Ae2p8NxH4WYLWNMc2AlsBX7\nnhHgVezV3d/g/6+/q/h7EwD/A2NMA+wJ0xDnMlNE3nZ+jmcApYFNQF9nTdhvZBP7UqActsKzGfi/\nNCdes95XfiV3pZRS+UdPqCqlVBDS5K6UUkFIk7tSSgUhTe5KKRWENLkrpVQQ0uSulFJBSJO7Um4y\nxrQyxszzdRxKuUOTu1Ke0QtDVEDQ5K6CjjGmj3PCg43GmM+do+ydN8Z8ZIzZZoxZbIwp6yx7hzFm\njXO0vW/TXPpdy1luszFmg/PqRoCSxpj/GWN2GmMm++xJKpUDTe4qqBhj6gE9gGYi0hh7qXwfoBiw\nXkRuw15aP9z5kInA35wjHW5Ls34q8IlzfTPs5PBgL78fCtwC1DLGNMv7Z6WU5/xumimlrtF9QGMg\nyjnIUhHs+DQO7LguAFOAb40xoUCYc3IQsIn+G+cgcVVFZC6AiFwBsLtjfcoIocaYzUAN4Jd8eF5K\neUSTuwo2BpgoIv9It9KY1zOUkzTlPZGQ5nYy+hlSfkqbZVSwWQp0NcaUh9RJqW/EzkfZ1VmmD7Da\nOV72GecoiAD9gBXOiSkOGWM6OfdR2Dn8qlIBQ2sdKqiIyE5jzGvAIufEDFeAIcBF7Kw2r2ObaXo4\nH9If+MKZvPcDTzrX9wO+NMa84dxHNzLTnjPKb+mQv6pAMMacF5GSvo5DqfyizTKqoNBajCpQtOau\nlFJBSGvuSikVhDS5K6VUENLkrpRSQUiTu1JKBSFN7kopFYQ0uSulVBD6f3/SWjCQedW7AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9e11e54110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEPCAYAAACJPZVzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VfWd//HXJ3tiVvY9bC51KeK4DKI2wGOKUFxmREEs\n2ulvrNNHOwKtddT5MWBr23FGmErn16qjIlUQxFZEgaq1jdYVrOICrgVkh7CFJZCE5Pv743tvchNI\ncm9I7r0nvJ+Px/dxzz335JxvLuR9v/f7Ped7zDmHiIgEQ0qiKyAiItFTaIuIBIhCW0QkQBTaIiIB\notAWEQkQhbaISIBEFdpmVmBmi83sYzNbY2YXtXfFRETkWGlRbnc/sNw5d62ZpQE57VgnERFpgrV0\ncY2Z5QPvOecGxadKIiLSlGi6RwYAu8xsrpm9a2YPmVl2e1dMRESOFU1opwHnAf/POXceUAHc0a61\nEhGR44qmT3szsMk5907o+dPAvzbeyMw0iYmISIyccxbL9i22tJ1zO4BNZnZaaNUoYG0T2wayzJgx\nI+F1UP0TXw/VP5glyPVvjWjPHrkVmG9m6cA64B9bdTQRETkhUYW2c+594IJ2rouIiLRAV0QCJSUl\nia7CCVH9E0v1T6yg1z9WLZ6nHfWOzFxb7UtE5GRgZrgYByKj7dMWkSj079+fL7/8MtHVkCRTXFzM\nhg0b2mRfammLtKFQyynR1ZAk09T/i9a0tNWnLSISIAptEZEAUWiLiASIQltEWvTll1+SkpJCbW0t\nAGPHjuXxxx+PattY/fznP+c73/lOq+va0Sm0RU4CY8aMYebMmcesf/bZZ+nZs2dUAWtWP162fPly\nJk+eHNW2zXnllVfo27dvg3V33nknDz30UFQ/H4t58+Zx6aWXtvl+402hLXISuOmmm3jiiSeOWf/E\nE08wefJkUlISEwXOuagDvi3E81jtRaEtchK4+uqr2b17N6+99lrdun379vH8889z4403Ar71fN55\n51FQUEBxcTF33313k/sbMWIEjz76KAC1tbXcdtttdO3alcGDB7Ns2bIG2z722GOceeaZ5OfnM3jw\n4LpWdEVFBWPHjmXr1q3k5eWRn5/P9u3bufvuuxu04pcuXcrZZ59Np06dGDlyJJ988kndawMGDGDW\nrFkMGTKEoqIirr/+eqqqqmJ+f7Zt28ZVV11F586dOe2003j44YfrXlu1ahUXXHABBQUF9OzZk9tu\nuw2AyspKJk+eTJcuXSgqKuKiiy6irKws5mPHSqEtchLIysri2muv5Te/+U3dukWLFvGVr3yFs88+\nG4Dc3Fwef/xxysvLWbZsGQ888ABLly5tcd8PPfQQy5cv5/333+edd97h6aefbvB69+7dWb58Ofv3\n72fu3LlMmzaN1atXk5OTw4oVK+jVqxcHDhxg//799OjRA6hvEX/22WdMmjSJOXPmUFZWxpgxY7ji\niis4evRo3f4XL17Miy++yPr163n//fd57LHHYn5/JkyYQL9+/di+fTuLFy/mrrvuorS0FIApU6Yw\ndepUysvL+etf/8p1110H+O6W/fv3s2XLFvbs2cMDDzxAdnb73x9GoS0SR2ZtU1rjpptuYvHixXUt\n0ccff5ybbrqp7vXLLruMs846C4Czzz6biRMn8sorr7S438WLFzN16lR69epFYWEhd955Z4PXx4wZ\nQ//+/QG49NJL+frXv86f//znqOr81FNPMW7cOEaOHElqaiq33XYbhw8f5o033qjbZsqUKXTv3p3C\nwkKuuOIKVq9eHdW+wzZv3sybb77JvffeS3p6OkOGDOGf/umf6j7g0tPT+eKLL9i9ezc5OTlceOGF\ndet3797NZ599hpkxdOhQcnNzYzp2ayi0ReLIubYprTF8+HC6du3KkiVLWLduHatWrWLSpEl1r69c\nuZKRI0fSrVs3CgsLefDBB9m1a1eL+926dWuDwcTi4uIGr69YsYJhw4bRuXNnioqKWLFiRVT7De87\ncn9mRt++fdmyZUvduu7du9ct5+TkcPDgwaj2HXmMTp06kZNTf7/y4uLiumM8+uijfPrpp5xxxhlc\ndNFFdd0/kydPZvTo0UycOJE+ffpwxx13UFNTE9OxW0OhLXISmTx5MvPmzeOJJ55g9OjRdO3ate61\nSZMmcfXVV7Nlyxb27dvHLbfcEtUl+T179mTTpk11zyPnXqmqqmL8+PHcfvvtlJWVsXfvXsaMGVO3\n35YGBnv16nXMXC6bNm2iT58+Uf2+0ejVqxd79uzh0KFDdes2btxI7969ARg0aBALFiygrKyM22+/\nnfHjx3P48GHS0tKYPn06a9as4Y033uC5555r0P3UXhTaIieRG2+8kT/84Q88/PDDDbpGAA4ePEhR\nURHp6emsXLmSBQsWNHi9qQC/7rrrmDNnDlu2bGHv3r3ce++9da9VVVVRVVVFly5dSElJYcWKFbz4\n4ot1r3fv3p3du3ezf//+Jve9bNky/vSnP3H06FHuu+8+srKyGDZsWKt+/9raWiorKxuUPn36cPHF\nF3PnnXdSWVnJBx98wCOPPFI3GDp//vy6bwYFBQWYGSkpKZSWlvLRRx9RW1tLbm4u6enpcTkLR6Et\nchIpLi7m4osvpqKigiuvvLLBa7/61a+YPn06BQUF3HPPPUyYMKHB65Gt4sjlm2++mdGjRzNkyBDO\nP/98rrnmmrrXcnNzmTNnDtdeey2dOnVi4cKFXHXVVXWvn3766Vx//fUMHDiQTp06sX379gbHPO20\n03jiiSf4/ve/T9euXVm2bBnPPfccaWlpx9QjGm+++SY5OTnk5OSQnZ1NTk4OtbW1LFiwgPXr19Or\nVy+uueYafvKTnzBixAgAfv/733PWWWeRn5/PtGnTWLRoEZmZmWzfvp3x48dTUFDAWWedxYgRI5o9\nd72taJY/kTakWf7keII5y19pKfz3f8ftcCIiHVH8QnvPHnj11bgdTkSkI4pfaBcVwd69cTuciEhH\nFL/QLiyEffvidjgRkY5ILW0RkQCJb2irpS0ickLiF9p5eXDoEMThMk8RkY4qfqGdkgL5+Wpti4ic\ngPheEanBSJEOoba2lry8PDZv3tym20rL4hvaGowUSYjwTQby8/NJTU0lJyenbt2TTz4Z8/5SUlI4\ncOBAVBM3xbJtrKZPn863v/3tNt9vMkuLZiMz2wCUA7VAtXPuwlYdTYORIglx4MCBuuWBAwfyyCOP\n1M2tcTw1NTWkpqbGo2oSo2hb2rVAiXNuaKsDG3z3iFraIgnlnDtmHozp06czceJEJk2aREFBAfPn\nz+ett95i2LBhFBUV0bt3b6ZMmVI3X3RNTQ0pKSls3LgR8FO+TpkyhbFjx5Kfn8/w4cPrplSNZVvw\n82+ffvrpFBUVceutt3LJJZe0asrTtWvXUlJSQlFREUOGDGH58uV1rz3//PN1t0Dr168f999/PwBl\nZWV84xvfoKioiM6dO1NSUhLzcdtbtKFtMWzbNLW0RZLWkiVL+OY3v0l5eTkTJkwgPT2dOXPmsGfP\nHl5//XVeeOEFHnzwwbrtG8+w9+STT/LTn/6UvXv30rdvX6ZPnx7ztjt37mTChAnMmjWLXbt2MWDA\nAFatWhXz71JdXc24ceO44oor2LVrF7Nnz2bChAmsW7cOgG9/+9vMnTuX/fv388EHH/C1r30NgP/6\nr/9i0KBB7N69mx07dnDPPffEfOz2Fm0QO+AFM1tlZje3+mhqacvJLpH3G2vBJZdcwtixYwHIzMzk\nb/7mb7jgggswM/r378/NN9/c4PZjjVvr48ePZ+jQoaSmpnLDDTc0uO1XtNsuW7aMoUOHMm7cOFJT\nU5k2bRqdO3eO+Xd5/fXXqa6u5oc//CGpqamMGjWKMWPGsHDhQgAyMjJYs2YNBw8epLCwkHPPPRfw\ntxDbunUrGzZsIC0tjUsuuSTmY7e3aEN7uHPufGAs8D0za91vooFIOdkl8n5jLYi8ZRjAp59+yrhx\n4+jZsycFBQXMmDGj2duEhW/KCy3f9qupbRvfugxo1QDm1q1b6devX4N1kbcQe+aZZ3j22Wfp168f\nI0eOZOXKlQDceeed9OvXj1GjRnHqqady3333xXzs9hbVQKRzblvosczMngEuBF5rvN3MmTPrlktK\nSo7tDyoqgojbEolI8mjchXHLLbcwbNgwFi9eTHZ2NrNmzaq7P2J76dmzZ4M72wAN7gcZrV69ejW4\nBRr4W4gNGTIEgAsuuIBnn32WmpoafvGLXzBx4kTWrVtHbm4us2fPZvbs2axZs4aSkhIuuugiLr30\n0tb/UhFKS0vr7vLeWi2GtpnlACnOuYNmdgrwdeDu420bGdrHpe4RkcA4cOAABQUFZGdn8/HHH/Pg\ngw+2y2l7kcaNG8fUqVNZtmwZl19+Ob/85S9bvAnw0aNHqaysrHtuZlx88cWkpaUxe/Zsbr31Vl59\n9VVWrFjBz372M44cOcIzzzzDuHHjyMvLIzc3t+5MmfAA5cCBA8nLyyMtLa1NbyHWuDF7993HjdJm\nRVOb7sBrZvYe8BbwnHPuxRZ+5vg0ECmScNHeomvWrFk89thj5Ofn893vfpeJEyc2uZ+W9hnttt26\ndWPRokVMmzaNLl26sH79eoYOHUpmZmaTPzN//vwGtxA744wzyMjIYOnSpSxZsoQuXbowdepUnnzy\nSQYNGgTAvHnz6N+/P4WFhcydO5f58+cDvkto5MiR5OXlcemllzJ16lSGDx/e7O8Wb/G93dhbb8GU\nKfD2221yTJFko9uNta3a2lp69erFb3/726QLz1gE83ZjoIFIEWnRCy+8QHl5OZWVlfz4xz8mIyOD\nCy9s/eUhHY3mHhGRpPLaa68xcOBAunfvzksvvcSSJUtIT09PdLWSRny7R6qqIDcXKivb7VxTkURS\n94gcT3C7RzIyID0dKirielgRkY4ivqENOu1PROQExD+0NRgpItJqUV0R2aY0GCkdWHFxcdTnQcvJ\no7i4uM32Ff/QVktbOrANGzYkugrSwSWme0QtbRGRVtFApIhIgGggUkQkQBLT0lb3iIhIq6ilLSIS\nIBqIFBEJEA1EiogEiLpHREQCRAORIiIBopa2iEiAxD+0c3PhyBGoro77oUVEgi7+oW2mLhIRkVaK\nf2iDukhERFopMaGtlraISKuopS0iEiBqaYuIBIha2iIiAZK40FZLW0QkZonrHlFLW0QkZuoeEREJ\nEA1EiogESNShbWYpZvaumS094aOqpS0i0iqxtLSnAGvb5KgaiBQRaZWoQtvM+gBjgYfb5KgaiBQR\naZVoW9r/DfwIcG1yVHWPiIi0SlpLG5jZN4AdzrnVZlYCWFPbzpw5s265pKSEkpKS429YUADl5eCc\nn/VPROQkUFpaSmlp6Qntw5xrvvFsZj8DvgkcBbKBPOB3zrkbG23nWtpXA3l5sGUL5OfHWmcRkQ7B\nzHDOxdRybbF7xDl3l3Oun3NuIDAR+GPjwG4VnfYnIhKzxJynDerXFhFphRb7tCM5514BXmmTIyu0\nRURilriWtrpHRERipu4REZEAUUtbRCRA1NIWEQkQhbaISICoe0REJEDU0hYRCRC1tEVEAkQtbRGR\nAFFoi4gEiLpHREQCJHGhnZMDR49CZWXCqiAiEjSJC20ztbZFRGKUuNAG9WuLiMQosaGtlraISEzU\n0hYRCRCFtohIgKh7REQkQNTSFhEJELW0RUQCRC1tEZEAUWiLiASIukdERAJELW0RkQBRS1tEJEDU\n0hYRCRBzzrXNjsxczPuqqYGMDKiuhpTEfn6IiMSbmeGcs1h+JrFJmZoKubmwf39CqyEiEhQthraZ\nZZrZ22b2npl9aGYz2rQG6iIREYlaWksbOOcqzWyEc67CzFKB181shXNuZZvUQIORIiJRi6p7xDlX\nEVrMxAd923SEg1raIiIxiCq0zSzFzN4DtgMvOedWtVkNCgsV2iIiUWqxewTAOVcLDDWzfGCJmZ3p\nnFvbeLuZM2fWLZeUlFBSUtLyzouK1D0iIieF0tJSSktLT2gfMZ/yZ2bTgUPOudmN1sd+yh/AD38I\nPXvCbbfF/rMiIgHWLqf8mVkXMysILWcDfwd80roqHocGIkVEohZN90hPYJ6ZpeBDfpFzbnmb1aCo\nCD7+uM12JyLSkUVzyt+HwHntVgMNRIqIRC3x145rIFJEJGrJEdpqaYuIRCXxoa2BSBGRqCU+tNXS\nFhGJWuJDOzwQ2UZTxIqIdGSJD+3sbDCDI0cSXRMRkaSX+NAGdZGIiEQpOUJbg5EiIlFJjtBWS1tE\nJCrJEdpqaYuIRCU5QlstbRGRqCRHaGv+ERGRqCRHaGv+ERGRqCRPaKulLSLSouQIbQ1EiohEJTlC\nWy1tEZGoJEdoayBSRCQqyRHaGogUEYlK8oS2WtoiIi1KjtDWQKSISFTMtdE81mbmWr2v2lpIT4eq\nKkhNbZP6iIgkOzPDOWex/ExytLRTUiA/X61tEZEWJEdogwYjRUSikFyhrcFIEZFmJU9oazBSRKRF\nyRPaammLiLQoeUJbV0WKiLQoeUJbA5EiIi1qMbTNrI+Z/dHM1pjZh2Z2a7vURC1tEZEWRdPSPgr8\nwDl3FjAM+J6ZndHmNVFLW0SkRS2GtnNuu3NudWj5IPAx0LvNa6KBSBGRFsXUp21m/YFzgbfbvCbq\nHhERaVFatBuaWS7wNDAl1OI+xsyZM+uWS0pKKCkpib4m6h4RkQ6utLSU0tLSE9pHVBNGmVka8Dyw\nwjl3fxPbtH7CKIBPPoErr4TPPmv9PkREAqQ9J4x6FFjbVGC3CbW0RURa1GJL28yGA68CHwIuVO5y\nzv2+0XYn1tKurITcXD89q8X0wSMiEkitaWknx3zaYTk5sHOnD28RkQ4uuPNph6mLRESkWckV2jrt\nT0SkWckV2mppi4g0K/lCWy1tEZEmJVdo60YIIiLNSq7QVktbRKRZyRXaGogUEWlWcoW2BiJFRJqV\nXKGtlraISLOSK7TV0hYRaVbyhbZa2iIiTUqu0Fb3iIhIs5IrtNU9IiLSrOQKbbW0RUSalVyhnZcH\nR45AdXWiayIikpSSK7TNdCm7iEgzor6x74l6/314/nno0aNh6dYN0tMjNgx3kXTtGq+qiYgERtxC\n2wwOHYLXX4ft2+tLWZnP6XCI/7z6q5w16htkT7gKrrgChg+HtLhVU0QkqSX8dmM1NbB7d32If/qJ\n4/mfvMf3+j3H2KNLSdu8AcaO9QF++eWQn98m9RURSbTg3yMypLwcpk+Hp56C+3+0mWuznyfl+aXw\n2mvwt3/rA/zv/x769GmT44nIya2qCt5+Gy64ALKy4nfcDhPaYX/5C3z3u5CRAb/+NZwz4CC89BIs\nXeo7yH/5S5g4sU2PKSIdX00NvPsu/PGPvrzxBpxxBixcCIMGxa8eHS60wb+5//u/8O//DjfdBDNm\nhG7Wvno1XHMNXHkl/Od/NhrNFBGp5xysWVMf0q+8Ar17w8iRvnzta/7avnjrkKEdtmMH/OhHUFoK\nv/iF7x2xfXvhxhv9KYJPPQU9e7bb8UUkOJyD9evh5Zfrgzo3tz6kR4zwJz4kWocO7bBXXvFdJoMH\nw6OPQpdOtfCzn/n+kyefhMsua/c6iEjy2batPqBffhkqK2HUqPqg7t8/0TU81kkR2uAHDf7t33zj\neuFCGDYMeOEF339y++0wbZo/x1BEOoTaWjh4EA4caFjKyvz5CS+/7M8+KynxAT1qlO+jTvYYOGlC\nO+zZZ+Hmm+Guu2DKFLAvN8D48TBwIDzyiL8sXkSSWkUFfPEFfP55w7JjR304V1RATo7/k44sRUX+\nhLJRo+DccyE1NdG/TWxOutAG32917bVQXOy7Swoyj8C//Iu/iud3v/MftyKSFD77DJ57Dj75pD6c\n9+zx7axTT/Xdnqee6kuvXvXhnJsLKck16UabOClDG3zf1Q9+4HtIFi+GoUPxLe077vCjl7fcAgUF\nCambyMlu2zbfjblgAWze7E8iOOec+nDu0yd4LeS20i6hbWaPAOOAHc65rzazXcJCO2zhQt/Ivuce\n+M53wNaugZ//HFasgG99y/eh9OuX0DqKBMmuXfDYY/485tNP92F79tn+C2xGRtM/V17uv+jOn++v\nt7jqKrjhBn/WhmalqNdeoX0JcBD4TbKHNsCnn/pu7SFD4IEHQud0b9wIc+bA3LkwejTcdhucd16i\nqyqSlJzzg3sPPgjLlvlLIUaP9v3OH34IH30EGzb4i1DOOae+nHmmnxhuwQJ/DdzIkTBpEowbB9nZ\nif6tklO7dY+YWTHwXBBCG/ygxfe+5y9Lvf12OP983zJIO1QODz/sT/Q+9VQf3pdf3jE7y0RitG8f\nPP64b+zU1MA//7O/DKJTp2O3PXIEPv7Yh3g4yD/6yPdJ33CDv+4tERerBI1Cu5FFi/wZJu+8A1u3\n+tb3+efD+UOqGbHrKXo/eR9WWekTfuhQOO006Nw5+c8Tkg6pqsqfKVFZWV+qqho+D5f9+33I7tvn\nuyLCy5HPwfcXNy69e/vH7t19e2XVKh/Uv/sdjBnjw/qyy/RnEA8K7WaUl/u5Bv7yFx/if/kLbN/m\n+D8D/sgNR+cxoOoTinZ9TkoK2Gmn+ZZ440fNMCjNqKnx3/LC5dCh+uWDB33/cFlZ/WO4hJ9XVPgz\nJTIz60tGRsPn4ZKf76c0Liz0Y+yRj+Fl52DLFl82bz627Nnjt83L82P13/qWn99e4ifhoT1jxoy6\n5yUlJZSUlMRSl7jbt88HeTjE31nlqNm5m7GDP+PSHp/z1azP6Ff1ObnbPsc+/9yfKDpgQH0ZOLB+\nuV8/zX8SYIcO+dNH162rf9yypWFrt6qqvkQ+P3zYB251tf8vkpMDp5xSvxx+3qWLv7dH+DFyuUsX\nH6DxbN1WVflzoXv3Vg9hvJSWllJaWlr3/O6772630O6PD+1zmtkmqVva0dqzp2Fr/J13fLifN9Rx\nWsEOivatp/P+9XTZv46uh9bTo2I9PY+so0v1Nnam9mRzWn8O5HSnqqAbtV26ktqjK5m9u5I7oCv5\ng7rS+Svd6HJaJ1LSUqis9Mfbvds/hkv4+b59vhXUo4f/KhsuPXr4Xhz9oR2rurq+ldtc2bzZB3M4\npMvL6z+Hw5/Fffr4AbSMjPoSbv2Gl9PT/TannOLXqUtBYtFeZ48sAEqAzsAOYIZzbu5xtusQoX08\nZWU+wPfuPf4fb0YGZKZUc8qeTWRs3UDF+h1UfFlG1dYy3M4yUveUkVleRu7hnRRUlZHn9nOAPCrI\noSo1m+q0HI5mZFOTmYPLyoGcbOyUHFJysjniMimvzGT/4Qz2VWSw91AGew5mcKAyg/TcTLILMsjM\nz/TJESp2Sg4pp2STcko2qXk5pOVlk56fjUtLp6bGXxIcfoxcrqnxpam+1Mh14E/dSk9v/tGsPsjC\ny42fg/8q39Jj4zA+ePDYMD561AdoS6V37/qAHjiwvn9XJJ5O2otrgqaq4iiVO8vJTT2MHTlc3/F5\nuNHyoUMNv5NHfDevOVzFkf2+VB88AhWH4YjfX8qRClIqD5NadZi0qgrSqg+TcbQCZylUpeVQnZZN\ndXr949H0HI6mZ1OdkUNNejYuMwuXmQkZmZCVhWVlQnYWKVmZWHYmKTlZuIwsqlMyqU7Nosr8Y6X5\ndUfIoioli0r8+pr0LGpS0nFYgxCOXG4c5sd7DHczRJbc3IbPMzPV2pXgUGhL05zzfQfhD4bGHxCR\ny5WV/pyuyMfjrYt8PN5yeF+Vlb4JnOk/BMjKarickeGb5S2VrCz/bSInp/6bxfGWw0kemei5uRpz\nkKSj0JbkFe53iQz2yA+Emhof7E2V6mq/feSHzvGWI/tQwv0n4ceUlPogjzzVIlyKiho+z83111dH\n9uukpBz7PDvbn85RUOCLPhwkSgptkaY457uWDh70JfJE5+OVvXv9SdPO+Q7/cH9O4+e1tf4Do7zc\n77O83Id2QUHDIC8srD9BOrL06uW/dchJSaEtkmjhEdNwgIfDfM8ef4VX45Olt23zLfxwiHfr1rDF\n39SyOu87BIW2SNDU1sLOnT7AN23ypyrt3duwxR/5GF6uqWl4Ini4Tz+y5Of7D4Fw6d694XN14ySc\nQlvkZBEeVI7sy48shw/7gC8r81fQ7NzpS3h51y5/EUC3bv4+XIMH15dBg/yJ6llZif4tOzyFtohE\np7bWd9ls3+6n7PvrX/00fuGycaO/iisc4v37+5mjwl00RUX1pbBQ8622kkJbRNrG0aM+uL/4wgf6\nl1/6bpnjlfJyf0ZOUZEP9k6d/CW74RL5vFMn303Tv//Je+eDCAptEYm/2lo/2Lp3b8O5GMLzMTRe\n3r7dd9GceqqfhDuyDB58UvW1K7RFJBgOHfJ3LFm7Ftas8Y9r1/rB2EGDfICfeir07esnYws/FhR0\nqLNmFNoiEmyHD/u7/65Z42fz2rjRB/nGjb5AwxDv27fhWTFdu/rH/PxAhLtCW0Q6Lud8/3k4xDdt\n8iV8Zky4lJX5q2wjQ7x79/o7QIQvcurd27+WwJnCFNoiIuBb7GVl9SG+bVv9HSHCd4XYssWfFtmz\nZ32Y33cfFBfHrZqtCW2dpyMiHU92tu8+6dev+e0qK32gh0O8sDA+9TsBammLiCRIa1ramvZdRCRA\nFNoiIgGi0BYRCRCFtohIgCi0RUQCRKEtIhIgCm0RkQBRaIuIBIhCW0QkQBTaIiIBotAWEQkQhbaI\nSIAotEVEAiSq0Dazy83sEzP7zMz+tb0rJSIix9diaJtZCvA/wGjgLOB6MzujvSsWT6WlpYmuwglR\n/RNL9U+soNc/VtG0tC8EPnfOfemcqwYWAle1b7XiK+j/6Kp/Yqn+iRX0+scqmtDuDWyKeL45tE5E\nROJMA5EiIgHS4u3GzOxvgZnOuctDz+8AnHPu3kbb6V5jIiIxavO7sZtZKvApMArYBqwErnfOfdza\nSoqISOu0eDd251yNmX0feBHfnfKIAltEJDHa7G7sIiLS/k54IDLoF96Y2QYze9/M3jOzlYmuT0vM\n7BEz22FmH0SsKzKzF83sUzN7wcwKElnH5jRR/xlmttnM3g2VyxNZx6aYWR8z+6OZrTGzD83s1tD6\nQLz/x6mWvDK4AAAFDUlEQVT/v4TWB+X9zzSzt0N/qx+a2YzQ+v5m9lYog540sxZ7EBKhmfrPNbN1\nofXvmtlXm92Rc67VBR/6XwDFQDqwGjjjRPYZ7wKsA4oSXY8Y6nsJcC7wQcS6e4HbQ8v/CvxHousZ\nY/1nAD9IdN2iqHsP4NzQci5+rOeMoLz/zdQ/EO9/qN45ocdU4C3gImARcG1o/a+BWxJdzxjrPxf4\nh2j3caIt7Y5w4Y0RoFMfnXOvAXsbrb4KmBdangdcHddKxaCJ+oP/d0hqzrntzrnVoeWDwMdAHwLy\n/jdR//A1F0n//gM45ypCi5n4MTkHjAB+G1o/D/j7BFQtKsepf23oedTv/4mGVUe48MYBL5jZKjO7\nOdGVaaVuzrkd4P8wgW4Jrk9rfM/MVpvZw8navRDJzPrjvzG8BXQP2vsfUf+3Q6sC8f6bWYqZvQds\nB14C/grsc86Fw28z0CtR9WtJ4/o751aFXron9P7PMrP05vYRmBZmOxrunDsfGIv/j3tJoivUBoI2\nuvwrYJBz7lz8f+bZCa5Ps8wsF3gamBJqsTZ+v5P6/T9O/QPz/jvnap1zQ/HfcC7Ed+8ERuP6m9mZ\nwB3Oua8AFwCd8V1sTTrR0N4C9It43ie0LjCcc9tCj2XAM/j/CEGzw8y6A5hZD2BngusTE+dcmQt1\n9AH/i//Pm5RCg1xPA487554NrQ7M+3+8+gfp/Q9zzu0HSoFhQGFoYjsISAZF1P/yiG9p1fj+7WYz\n6ERDexUw2MyKzSwDmAgsPcF9xo2Z5YRaHZjZKcDXgY8SW6uoGA37wJYC3wot3wQ82/gHkkyD+oeC\nLuwfSO5/g0eBtc65+yPWBen9P6b+QXn/zaxLuOvGzLKBvwPWAn8Crg1tlrTvfxP1/yT8/puZ4cdD\nmn3/T/g87dDpQfdTf+HNf5zQDuPIzAbgW9cOPygwP9nrb2YLgBL816gd+JH/JcBioC/wJXCdc25f\nourYnCbqPwLfv1oLbMCP/u9IUBWbZGbDgVeBD/H/ZxxwF/4q4adI8ve/mfpPIhjv/zn4gcaUUFnk\nnPtp6O94IVAEvAd8M9RqTSrN1P9loAu+IbMa+OeIActj93OioS0iIvGjgUgRkQBRaIuIBIhCW0Qk\nQBTaIiIBotAWEQkQhbaISIAotOWkZ2ZfM7PnEl0PkWgotEU8XbAggaDQlsAwsxtCk8i/a2a/Ds2Y\ndsDMZpvZR2b2kpl1Dm17rpm9GZo57bcRlw8PCm232szeCV1NB5BnZovN7GMzezxhv6RICxTaEghm\ndgYwAbjYOXce/pLrG4AcYKVz7mz8JdozQj8yD/hRaOa6jyLWzwd+GVp/Mf5m1eAv474VOBMYZGYX\nt/9vJRK7pLwtj8hxjALOA1aFJtbJws9dUouf9wPgCeC3ZpYPFIRuuAA+wJ8KTQ7W2zm3FMA5VwXg\nd8fK8IyPZrYa6A+8EYffSyQmCm0JCgPmOef+rcFKs+mNtnMR28eiMmK5Bv1tSJJS94gExcvAeDPr\nCnU30+2Hv9fe+NA2NwCvheYq3hOa1Q5gMvBKaML/TWZ2VWgfGaEpMkUCQ60JCQTn3Mdm9n+BF0MT\n3lcB3wcO4e8AMh3fXTIh9CM3AQ+GQnkd8I+h9ZOBh8zsx6F9XMuxdCaJJC1NzSqBZmYHnHN5ia6H\nSLyoe0SCTq0OOamopS0iEiBqaYuIBIhCW0QkQBTaIiIBotAWEQkQhbaISIAotEVEAuT/A8bsENyU\n6hz+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9df5be4e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "log = h5py.File('Training_logs_DMN_plus.h5','r+') # Loading logs about change of training and validation loss and accuracy over epochs\n",
    "\n",
    "y1 = log['val_acc'][...]\n",
    "y2 = log['acc'][...]\n",
    "\n",
    "x = np.arange(1,len(y1)+1,1) # (1 = starting epoch, len(y1) = no. of epochs, 1 = step) \n",
    "\n",
    "plt.plot(x,y1,'b',label='Validation Accuracy') \n",
    "plt.plot(x,y2,'r',label='Training Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()\n",
    "\n",
    "y1 = log['val_loss'][...]\n",
    "y2 = log['loss'][...]\n",
    "\n",
    "plt.plot(x,y1,'b',label='Validation Loss')\n",
    "plt.plot(x,y2,'r',label='Training Loss')\n",
    "plt.legend(loc='upper right')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading pre-trained weights for the model...\n",
      "INFO:tensorflow:Restoring parameters from DMN_Model_Backup/model.ckpt\n",
      "\n",
      "RESTORATION COMPLETE\n",
      "\n",
      "Testing Model Performance...\n",
      "\n",
      "Test Loss= 0.941, Test Accuracy= 51.000%\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess: # Begin session\n",
    "    \n",
    "    print 'Loading pre-trained weights for the model...'\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, 'DMN_Model_Backup/model.ckpt')\n",
    "    sess.run(tf.global_variables())\n",
    "    print '\\nRESTORATION COMPLETE\\n'\n",
    "    \n",
    "    print 'Testing Model Performance...'\n",
    "    \n",
    "    total_test_loss = 0\n",
    "    total_test_acc = 0\n",
    "    \n",
    "    test_batch_size = 100 #(should be able to divide total no. of test samples without remainder)\n",
    "    batches_test_fact_stories,batches_test_questions,batches_test_answers = create_batches(test_fact_stories,test_questions,test_answers,test_batch_size)\n",
    "        \n",
    "    for i in xrange(len(batches_test_questions)):\n",
    "        test_loss, test_acc = sess.run([cost, accuracy], \n",
    "                                        feed_dict={tf_facts: batches_test_fact_stories[i], \n",
    "                                                   tf_questions: batches_test_questions[i], \n",
    "                                                   tf_answers: batches_test_answers[i],\n",
    "                                                   training: False})\n",
    "        total_test_loss += test_loss\n",
    "        total_test_acc += test_acc\n",
    "                      \n",
    "            \n",
    "    avg_test_loss = total_test_loss/len(batches_test_questions) \n",
    "    avg_test_acc = total_test_acc/len(batches_test_questions) \n",
    "\n",
    "\n",
    "    print \"\\nTest Loss= \" + \\\n",
    "          \"{:.3f}\".format(avg_test_loss) + \", Test Accuracy= \" + \\\n",
    "          \"{:.3f}%\".format(avg_test_acc*100)+\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}