12_Grid_n_Randomised_Search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets \n",
    "iris = datasets.load_iris()\n",
    "\n",
    "X = iris.data[:,2:]\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, cross_val_score \n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y,random_state=7)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Instantiate a nearest neighbour classifier \n",
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "knn_classifier = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Preparing a parameter grid - a dictionary with the parameters setting to try with grid search\n",
    "\n",
    "param_grid = {'n_neighbors': list(range(3,9,1))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#instantiating a grid search \n",
    "from sklearn.model_selection import GridSearchCV \n",
    "grid_search = GridSearchCV(knn_classifier, param_grid, cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, estimator=KNeighborsClassifier(),\n",
       "             param_grid={'n_neighbors': [3, 4, 5, 6, 7, 8]})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 3}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#View the results \n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.95530303, 0.93787879, 0.95530303, 0.95530303, 0.95530303,\n",
       "       0.95530303])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.cv_results_['mean_test_score']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Another way - brute force methods (time consuming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 0.9553030303030303),\n",
       " (5, 0.9553030303030303),\n",
       " (6, 0.9553030303030303),\n",
       " (7, 0.9553030303030303),\n",
       " (8, 0.9553030303030303),\n",
       " (4, 0.9378787878787879)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scores = []\n",
    "\n",
    "for n_neighbors in range(3,9,1):\n",
    "    knn_classifier = KNeighborsClassifier(n_neighbors=n_neighbors)\n",
    "    all_scores.append((n_neighbors, cross_val_score(knn_classifier, X_train, y_train, cv=10).mean()))\n",
    "\n",
    "sorted(all_scores, key=lambda x:x[1], reverse=True)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Randomized Search\n",
    "##### `RandomizedSearchCV` is more useful with medium amount of data or with a model involving a few parameters, as `GridSearchCV` can be computationally expensive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets \n",
    "iris = datasets.load_iris()\n",
    "X = iris.data[:,2:]\n",
    "y = iris.target \n",
    "\n",
    "from sklearn.model_selection import train_test_split \n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,stratify=y,random_state=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "knn_clf = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Parameter distribution \n",
    "\n",
    "param_dist = {'n_neighbors':list(range(3,9,1))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV \n",
    "rs = RandomizedSearchCV(knn_clf, param_dist, cv=10, n_iter=6)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=10, estimator=KNeighborsClassifier(), n_iter=6,\n",
       "                   param_distributions={'n_neighbors': [3, 4, 5, 6, 7, 8]})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fitting the randomized grid search estimator\n",
    "rs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 3}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Results \n",
    "rs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<zip at 0x23621996cc0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(rs.cv_results_['params'],rs.cv_results_['mean_test_score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 15}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#If we scan a larger parameter space with RandomizedSearchCV\n",
    "\n",
    "param_dist = {'n_neighbors': list(range(3,50,1))}\n",
    "rs = RandomizedSearchCV(knn_clf, param_dist, cv=10, n_iter=15)\n",
    "rs.fit(X_train, y_train)\n",
    "rs.best_params_"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Compare timing of RandomizedSearchCV and GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 141 ms\n",
      "Wall time: 210 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=10, estimator=KNeighborsClassifier(), n_iter=15,\n",
       "                   param_distributions={'n_neighbors': [3, 4, 5, 6, 7, 8, 9, 10,\n",
       "                                                        11, 12, 13, 14, 15, 16,\n",
       "                                                        17, 18, 19, 20, 21, 22,\n",
       "                                                        23, 24, 25, 26, 27, 28,\n",
       "                                                        29, 30, 31, 32, ...]})"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RandomizedSearchCV\n",
    "%time rs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#GridSearchCV\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "param_grid = {'n_neighbors':list(range(3,50,1))}\n",
    "gs = GridSearchCV(knn_clf, param_grid, cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 172 ms\n",
      "Wall time: 522 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, estimator=KNeighborsClassifier(),\n",
       "             param_grid={'n_neighbors': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,\n",
       "                                         14, 15, 16, 17, 18, 19, 20, 21, 22, 23,\n",
       "                                         24, 25, 26, 27, 28, 29, 30, 31, 32, ...]})"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#timing grid search\n",
    "%time gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 11}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}