data_prep_scrapped_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%cd /Users/Deepthi/Documents/DSE/Capstone/Data/From Julius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%cd ./../scraper/\n",
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_cpp_br_pr = pd.read_csv('reviews_Women_all_final_scraped.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_all = pd.read_hdf('womens_Meta_scraped.hdf5', key='data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn.feature_extraction.text \n",
    "from sklearn.feature_extraction.text  import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "product_title = list(womens_scrapped_all['NAME'].drop_duplicates().dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_stop_words = sklearn.feature_extraction.text.ENGLISH_STOP_WORDS.union({'women','womens'}) \n",
    "\n",
    "corpus = product_title\n",
    "\n",
    "## Considered only 2-grams after trying 1-grams and 3-grams. 2-grams give more meaningful information\n",
    "vectorizer = TfidfVectorizer(stop_words = set(my_stop_words),ngram_range=(2,2)) \n",
    "\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "idf = vectorizer.idf_\n",
    "\n",
    "# print dict(zip(vectorizer.get_feature_names(), idf))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "z=vectorizer.inverse_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bi_grams = list(z[0])\n",
    "for i in range(1,len(z)):\n",
    "    bi_grams = bi_grams + list(z[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# list(z[0]) + list(z[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "d= Counter(bi_grams)\n",
    "\n",
    "df = pd.DataFrame.from_dict(d, orient='index').reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.columns = [['bigrams','count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.sort_values('count',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(df[df['count']>2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df[df['count']>2]['bigrams']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sel_bigrams=list(df[df['count']>2]['bigrams'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(sel_bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(sel_bigrams)):\n",
    "    womens_scrapped_all[sel_bigrams[i]]=womens_scrapped_all['NAME'].fillna('0').str.lower()\\\n",
    "                                      .str.contains(sel_bigrams[i]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# len(womens_scrapped_all.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i in range(1,10+1):\n",
    "#     if i < 10:\n",
    "#         print i\n",
    "#     else:\n",
    "#         print 'next number'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Create a feature vector field by concatenating each of the column generated using the 2-grams\n",
    "womens_scrapped_all['feature_vector'] = womens_scrapped_all[sel_bigrams].transpose().apply(lambda x: np.array(x).tobytes())\n",
    "\n",
    "# old version below\n",
    "# womens_scrapped_all['feature_vector'] = \"'\"+womens_scrapped_all[sel_bigrams[0]].astype(str)\n",
    "\n",
    "# for i in range(1,len(sel_bigrams)+1):\n",
    "#     if i < len(sel_bigrams):\n",
    "#         womens_scrapped_all['feature_vector'] = womens_scrapped_all['feature_vector'].astype(str)\\\n",
    "#                                             +womens_scrapped_all[sel_bigrams[i]].astype(str)\n",
    "#     else:\n",
    "#         womens_scrapped_all['feature_vector']=womens_scrapped_all['feature_vector']+\"'\"\n",
    "\n",
    "# df_meta_only_womens['feature_vector']=df_meta_only_womens[0].str.cat(df_meta_only_womens[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Create a feature vector field by concatenating each of the column generated using the 2-grams\n",
    "\n",
    "womens_scrapped_all['cnt'] = womens_scrapped_all[sel_bigrams[0]].astype(int)\n",
    "\n",
    "for i in range(1,len(sel_bigrams)):\n",
    "    womens_scrapped_all['cnt'] = womens_scrapped_all['cnt'].astype(int)\\\n",
    "                                        +womens_scrapped_all[sel_bigrams[i]].astype(int)\n",
    "\n",
    "# df_meta_only_womens['feature_vector']=df_meta_only_womens[0].str.cat(df_meta_only_womens[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# womens_scrapped_all['cnt']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_all.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_all[['asin','feature_vector']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i in range(len(womens_scrapped_all['feature_vector'])):\n",
    "#     if (len(womens_scrapped_all['feature_vector'][i])) == 0:\n",
    "#         print 1\n",
    "# #     else:\n",
    "# #         print 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# womens_scrapped_cpp_br_pr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_cpp_br_pr = womens_scrapped_cpp_br_pr.sort_values(by='asin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_all = womens_scrapped_all.sort_values(by='asin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# womens_scrapped_cpp_br_pr_for_merge=womens_scrapped_cpp_br_pr[womens_scrapped_cpp_br_pr['asin'].isin(list(womens_scrapped_all['asin']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# womens_scrapped_cpp_br_pr_for_merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final=womens_scrapped_cpp_br_pr.merge(womens_scrapped_all[['asin','feature_vector']], on='asin', how='left')\\\n",
    "[['reviewerID','asin','unixReviewTime','brand','price','feature_vector']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# final = pd.merge(womens_scrapped_cpp_br_pr,womens_scrapped_all[['asin','feature_vector']], on='asin',how='left')\\\n",
    "# [['reviewerId','asin','unixReviewTime','brand','price','feature_vector']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final = final.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_cpp_br_pr.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# womens_scrapped_all.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 190695-183973"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "womens_scrapped_cpp_br_pr.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final.to_csv('reviews_Women_scraped_all_cpp_fv_2.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}