week-3-project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Toronto Neighborhoods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from geopy.geocoders import Nominatim\n",
    "import folium"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- will use 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "- check for status code and headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n",
      "200\n",
      "{'Date': 'Sat, 22 Dec 2018 22:54:20 GMT', 'Content-Type': 'text/html; charset=UTF-8', 'Content-Length': '15354', 'Connection': 'keep-alive', 'Server': 'mw1320.eqiad.wmnet', 'X-Content-Type-Options': 'nosniff', 'P3P': 'CP=\"This is not a P3P policy! See https://en.wikipedia.org/wiki/Special:CentralAutoLogin/P3P for more info.\"', 'X-Powered-By': 'HHVM/3.18.6-dev', 'Content-language': 'en', 'Last-Modified': 'Fri, 07 Dec 2018 05:21:50 GMT', 'Backend-Timing': 'D=110520 t=1545369710028179', 'Vary': 'Accept-Encoding,Cookie,Authorization,X-Seven', 'Content-Encoding': 'gzip', 'X-Varnish': '120822320 116111504, 754087610 335184103', 'Via': '1.1 varnish (Varnish/5.1), 1.1 varnish (Varnish/5.1)', 'Age': '149550', 'X-Cache': 'cp1081 hit/9, cp1075 hit/15', 'X-Cache-Status': 'hit-front', 'Server-Timing': 'cache;desc=\"hit-front\"', 'Strict-Transport-Security': 'max-age=106384710; includeSubDomains; preload', 'Set-Cookie': 'WMF-Last-Access=22-Dec-2018;Path=/;HttpOnly;secure;Expires=Wed, 23 Jan 2019 12:00:00 GMT, WMF-Last-Access-Global=22-Dec-2018;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Wed, 23 Jan 2019 12:00:00 GMT, GeoIP=US:IL:Barrington:42.16:-88.14:v4; Path=/; secure; Domain=.wikipedia.org', 'X-Analytics': 'ns=0;page_id=539066;https=1;nocookies=1', 'X-Client-IP': '73.72.63.152', 'Cache-Control': 'private, s-maxage=0, max-age=0, must-revalidate', 'Accept-Ranges': 'bytes'}\n"
     ]
    }
   ],
   "source": [
    "url = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "result = requests.get(url)\n",
    "print(url)\n",
    "print(result.status_code)\n",
    "print(result.headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- will get data + clean it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(result.content, 'html.parser')\n",
    "table = soup.find('table')\n",
    "trs = table.find_all('tr')\n",
    "rows = []\n",
    "for tr in trs:\n",
    "    i = tr.find_all('td')\n",
    "    if i:\n",
    "        rows.append(i)\n",
    "        \n",
    "lst = []\n",
    "for row in rows:\n",
    "    postalcode = row[0].text.rstrip()\n",
    "    borough = row[1].text.rstrip()\n",
    "    neighborhood = row[2].text.rstrip()\n",
    "    if borough != 'Not assigned':\n",
    "        if neighborhood == 'Not assigned':\n",
    "            neighborhood = borough\n",
    "        lst.append([postalcode, borough, neighborhood])\n",
    "\n",
    "# lst"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- convert into a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(212, 3)\n"
     ]
    }
   ],
   "source": [
    "cols = ['PostalCode', 'Borough', 'Neighborhood']\n",
    "df = pd.DataFrame(lst, columns=cols)\n",
    "print(df.shape)\n",
    "# df[df.duplicated(['PostalCode'], keep=False)] - this would have shown the duplicate PostalCodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- custom groupby / agg to merge Neighborhoods\n",
    "    - groupby PostalCode, keep the first Borough and join() Neighborhoods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.groupby('PostalCode').agg(\n",
    "    {\n",
    "        'Borough':'first', \n",
    "        'Neighborhood': ', '.join,}\n",
    "    ).reset_index()\n",
    "\n",
    "# could have done:\n",
    "# df = df.groupby(['Postcode', 'Borough'], as_index=False).agg(lambda neighborhoods: ', '.join(neighborhoods))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- will check if 'M5A' example is done correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Harbourfront, Regent Park</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PostalCode           Borough               Neighborhood\n",
       "53        M5A  Downtown Toronto  Harbourfront, Regent Park"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['PostalCode'] == 'M5A']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- will check the df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}