Final_IST_Bigdata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parts of this code was used from Harper He Code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We can view .ipynb file contents right here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "# import os\n",
    "\n",
    "# # note -- we can look at a .ipynb file as a plain text file right here in Jupyter notebook\n",
    "# with open ('Lending_Club.ipynb', mode = 'r', encoding = 'utf-8') as f:\n",
    "#     a = json.load (f)\n",
    "\n",
    "# print(type(a))\n",
    "\n",
    "# print(a.keys())\n",
    "\n",
    "# #print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating list of notebooks to merge\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # generate list of files to be merged\n",
    "# notebooks_to_merge = [file for file in os.listdir(os.getcwd()) if file.endswith('.ipynb')]\n",
    "\n",
    "# # list should be in the order in which you want the notebooks to be merged\n",
    "# # my notebooks are labeled 1.ipynb, 2.ipynb... in the order I want\n",
    "# notebooks_to_merge.sort()\n",
    "\n",
    "# print(notebooks_to_merge)\n",
    "\n",
    "# del notebooks_to_merge[2:4]\n",
    "# print(notebooks_to_merge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate combined .ipynb file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def combine_ipynb_files(list_of_notebooks, combined_file_name):\n",
    "    \n",
    "#     '''\n",
    "#     parameters:\n",
    "#     `list_of_notebooks` is an ordered list of your .ipynb files to be merged\n",
    "#     `combined_file_name` is the name of your combined .ipynb file which will be generated by this function\n",
    "    \n",
    "#     returns: the filepath of the new file\n",
    "#     '''\n",
    "    \n",
    "#     with open (notebooks_to_merge[0], mode = 'r', encoding = 'utf-8') as f:\n",
    "#         a = json.load (f)\n",
    "    \n",
    "#     for notebook in notebooks_to_merge[1:]:\n",
    "#         with open (notebook, mode = 'r', encoding = 'utf-8') as f:\n",
    "#             b = json.load(f)\n",
    "#             a['cells'].extend (b['cells']) \n",
    "#                 # extend here, not append, so that each dictionary in b['cells']\n",
    "#                 # is added to new dictionary in a['cells']\n",
    "\n",
    "#     with open(combined_file_name, mode='w', encoding='utf-8') as f:\n",
    "#         json.dump(a, f)\n",
    "    \n",
    "#     print('Generated file: \"{}\".'.format(combined_file_name))\n",
    "    \n",
    "#     return (os.path.realpath(combined_file_name))\n",
    "\n",
    "# combine_ipynb_files(notebooks_to_merge, \"combined.ipynb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot run multiple SparkContexts at once; existing SparkContext(app=pyspark-shell, master=local[*]) created by __init__ at <ipython-input-5-02af4abe42c5>:24 ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-34-02af4abe42c5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[0mconfig\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSparkConf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msetAll\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'spark.executor.memory'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'8g'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'spark.executor.cores'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'3'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'spark.cores.max'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'3'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'spark.driver.memory'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'8g'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;31m# sc.stop()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m \u001b[0msc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[0mspark\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSparkSession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetOrCreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\context.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)\u001b[0m\n\u001b[0;32m    125\u001b[0m                 \" is not allowed as it is a security risk.\")\n\u001b[0;32m    126\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 127\u001b[1;33m         \u001b[0mSparkContext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgateway\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgateway\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    128\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    129\u001b[0m             self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\context.py\u001b[0m in \u001b[0;36m_ensure_initialized\u001b[1;34m(cls, instance, gateway, conf)\u001b[0m\n\u001b[0;32m    333\u001b[0m                         \u001b[1;34m\" created by %s at %s:%s \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m                         % (currentAppName, currentMaster,\n\u001b[1;32m--> 335\u001b[1;33m                             callsite.function, callsite.file, callsite.linenum))\n\u001b[0m\u001b[0;32m    336\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m                     \u001b[0mSparkContext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_active_spark_context\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minstance\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Cannot run multiple SparkContexts at once; existing SparkContext(app=pyspark-shell, master=local[*]) created by __init__ at <ipython-input-5-02af4abe42c5>:24 "
     ]
    }
   ],
   "source": [
    "# import packages\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkConf,SparkContext\n",
    "from pyspark.ml import feature\n",
    "from pyspark.ml import classification\n",
    "from pyspark.sql import functions as fn\n",
    "from pyspark.sql.functions import isnan, when, count, col\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator, \\\n",
    "    MulticlassClassificationEvaluator, \\\n",
    "    RegressionEvaluator\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "from pyspark.sql.functions import row_number, monotonically_increasing_id\n",
    "from pyspark.sql import Window\n",
    "\n",
    "config = SparkConf().setAll([('spark.executor.memory', '8g'), ('spark.executor.cores', '3'), ('spark.cores.max', '3'), ('spark.driver.memory','8g')])\n",
    "# sc.stop()\n",
    "sc = SparkContext(conf=config)\n",
    "\n",
    "spark = SparkSession.builder.config(conf=config).getOrCreate()\n",
    "sc.getConf().getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# expand the output display to see more columns of a pandas DataFrame\n",
    "pd.set_option('display.max_columns', None)  \n",
    "pd.set_option('display.expand_frame_repr', False)\n",
    "pd.set_option('max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is a function for checking the shape and size\n",
    "def size_shape(df):\n",
    "    print(\"rows:\",df.count())\n",
    "    print(\"columns:\",len(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the dataset\n",
    "df = spark.read.csv('loan.csv', header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a look at the first 10 rows\n",
    "# df_pd = df.limit(10).toPandas()\n",
    "# display(df_pd.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check how many records for each loan status\n",
    "# df_bystatus=df.groupby(df.loan_status).count()\n",
    "# df_bystatus.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show all the column names\n",
    "# df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the shape of df\n",
    "# print(\"rows:\",df.count())\n",
    "# print(\"columns:\",len(df.columns))\n",
    "\n",
    "# size_shape(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+------+\n",
      "|    loan_status| count|\n",
      "+---------------+------+\n",
      "|Not Charged Off|945329|\n",
      "|    Charged Off|261654|\n",
      "+---------------+------+\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# check the number of Nan or Null for each column\n",
    "#As per our objective, Filtering only the columns required\n",
    "current_df = df.filter(col('loan_status').isin(['Late (31-120 days)','Charged Off','Late (16-30 days)','Current']))\n",
    "#Renaming the Late values into a single late\n",
    "current_df = current_df.withColumn(\"loan_status\", \\\n",
    "              when(current_df[\"loan_status\"].isin(['Late (31-120 days)','Late (16-30 days)','Current']),'Not Charged Off').otherwise(current_df[\"loan_status\"]))\n",
    "\n",
    "#Our dataset with only the desired columns values (Charged off, Not Charged Off)\n",
    "print(current_df.groupby(current_df.loan_status).count().show())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Contains the nas present in all the columns in the dataset. We are planning to remove those coluomns with more than 50%Nas\n",
    "# checkna_pd = current_df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in current_df.columns]).toPandas()\n",
    "# checkna_pd\n",
    "\n",
    "# current_df.select([c for c in checkna_pd.columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://stackoverflow.com/questions/51322445/how-to-drop-all-columns-with-null-values-in-a-pyspark-dataframe - include in our references\n",
    "\n",
    "#Creating a dataframe which has the count of Nas of each column\n",
    "checkna = current_df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in current_df.columns])\n",
    "#Selecting the columns with NA values not greater than 50%\n",
    "No_of_rows = current_df.count()\n",
    "checkna_greater_than_50 = checkna.select([when(fn.col(c)<(No_of_rows*0.50),c).alias(c) for c in checkna.columns])\n",
    "\n",
    "\n",
    "def drop_null_columns(df):\n",
    "    \"\"\"\n",
    "    This function drops all columns which contain null values.\n",
    "    :param df: A PySpark DataFrame\n",
    "    \"\"\"\n",
    "    null_counts = df.select([fn.count(fn.when(fn.col(c).isNull()|isnan(fn.col(c)), c)).alias(c) for c in df.columns]).collect()[0].asDict()\n",
    "    to_drop = [k for k, v in null_counts.items() if v > 0]\n",
    "    df = df.drop(*to_drop)\n",
    "    return df\n",
    "\n",
    "# Using the function to find the list of columns with less than 50%Na values\n",
    "final_cols = drop_null_columns(checkna_greater_than_50)\n",
    "current_df1 = current_df.select([fn.col(c) for c in final_cols.columns])\n",
    "\n",
    "#selecting our dataframe with the final cols\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Benji Explore Columns 1-25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1=current_df1.select(current_df1.columns[0:25])\n",
    "#part1.columns\n",
    "# part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping columns that have date or will not use in analysis\n",
    "# drop the cols\n",
    "drop_var=['issue_d','zip_code','earliest_cr_line']\n",
    "part1=part1.drop(*drop_var)\n",
    "# print(size_shape(part1))\n",
    "# part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the summary of each col\n",
    "# part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the missing values\n",
    "# checkna_part1 = part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns])\n",
    "# checkna_part1.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns with missing values | the number of missing values | explanation of the type of data\n",
    "emp_title      | 105,247 | The job title supplied by the Borrower when applying for the loan.*\n",
    "\n",
    "title          | 12,520 | The loan title provided by the borrower\n",
    "\n",
    "addr_state     | 1 | The state provided by the borrower in the loan application\n",
    "\n",
    "dti            | 1,456 | A ratio calculated using the borrower’s total monthly debt payments on the total debt obligations, excluding mortgage and the requested LC loan, divided by the borrower’s self-reported monthly income.\n",
    "\n",
    "delinq_2yrs    | 1 | The number of 30+ days past-due incidences of delinquency in the borrower's credit file for the past 2 years\n",
    "\n",
    "inq_last_6mths | 1 | The number of inquiries in past 6 months (excluding auto and mortgage inquiries)\n",
    "\n",
    "open_acc       | 5 | The number of open credit lines in the borrower's credit file.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets look at the columns thave null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(part1.groupby(part1.emp_title).count().show())\n",
    "# print(part1.groupby(part1.addr_state).count().show())\n",
    "# print(part1.groupby(part1.dti).count().show())\n",
    "# print(part1.groupby(part1.delinq_2yrs).count().show())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(part1.groupby(part1.inq_last_6mths).count().show())\n",
    "# print(part1.groupby(part1.open_acc).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHeck the columns to see if they are categories\n",
    "# print(part1.groupby(part1.purpose).count().show())\n",
    "# print(part1.groupby(part1.pymnt_plan).count().show())\n",
    "# print(part1.groupby(part1.verification_status).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the columns to see if they are categories\n",
    "# print(part1.groupby(part1.term).count().show())\n",
    "# print(part1.groupby(part1.grade).count().show())\n",
    "# print(part1.groupby(part1.sub_grade).count().show())\n",
    "# print(part1.groupby(part1.home_ownership).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the columns into categories\n",
    "\n",
    "cat_col=['term','grade','sub_grade','home_ownership','verification_status','loan_status','pymnt_plan']\n",
    "part1_cat_col=part1.select(cat_col)\n",
    "# part1_cat_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we had columns that had bad data but casting the correct type should solve issue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After looking at the values, we have some more columns to drop\n",
    "drop_var_2=['emp_title',\"title\",'purpose','emp_length','addr_state']\n",
    "part1=part1.drop(*drop_var_2)\n",
    "# print(size_shape(part1))\n",
    "# part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at changing the numerical cols to the right type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1_num_col=part1.drop(*cat_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1_num_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1 = part1.withColumn('loan_amnt', part1['loan_amnt'].cast(FloatType()))\n",
    "part1 = part1.withColumn('funded_amnt', part1['funded_amnt'].cast(FloatType()))\n",
    "part1 = part1.withColumn('funded_amnt_inv', part1['funded_amnt_inv'].cast(FloatType()))\n",
    "part1 = part1.withColumn('int_rate',part1['int_rate'].cast(FloatType()))\n",
    "part1 = part1.withColumn('installment', part1['installment'].cast(FloatType()))\n",
    "part1 = part1.withColumn('annual_inc', part1['annual_inc'].cast(FloatType()))\n",
    "part1 = part1.withColumn('dti', part1['dti'].cast(FloatType()))\n",
    "part1 = part1.withColumn('delinq_2yrs', part1['delinq_2yrs'].cast(IntegerType()))\n",
    "part1 = part1.withColumn('inq_last_6mths', part1['inq_last_6mths'].cast(IntegerType()))\n",
    "part1 = part1.withColumn(\"open_acc\",part1[\"open_acc\"].cast(IntegerType()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns with missing values | the number of missing values | explanation of the type of data\n",
    "\n",
    "dti            | 1,489 | A ratio calculated using the borrower’s total monthly debt payments on the total debt obligations, excluding mortgage and the requested LC loan, divided by the borrower’s self-reported monthly income.\n",
    "\n",
    "delinq_2yrs    | 31 | The number of 30+ days past-due incidences of delinquency in the borrower's credit file for the past 2 years\n",
    "\n",
    "inq_last_6mths | 24 | The number of inquiries in past 6 months (excluding auto and mortgage inquiries)\n",
    "\n",
    "open_acc       | 23 | The number of open credit lines in the borrower's credit file.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets fill in the values that are missing with the mean value\n",
    "\n",
    "dti_mean=part1.select(fn.avg('dti')).collect()\n",
    "part1 = part1.na.fill({'dti':dti_mean[0][0]})\n",
    "\n",
    "delinq_2yrs_mean=part1.select(fn.avg('delinq_2yrs')).collect()\n",
    "part1 = part1.na.fill({'delinq_2yrs':delinq_2yrs_mean[0][0]})\n",
    "\n",
    "inq_last_6mths_mean=part1.select(fn.avg('inq_last_6mths')).collect()\n",
    "part1 = part1.na.fill({'inq_last_6mths':inq_last_6mths_mean[0][0]})\n",
    "\n",
    "open_acc_mean=part1.select(fn.avg('open_acc')).collect()\n",
    "part1 = part1.na.fill({'open_acc':open_acc_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv  term  int_rate  installment  grade  sub_grade  home_ownership  annual_inc  verification_status  loan_status  pymnt_plan  dti  delinq_2yrs  inq_last_6mths  open_acc\n",
       "0  0          0            0                0     0         0            0      0          0               0           0                    0            0           0    0            0               0       "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recheck for missing values\n",
    "part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# size_shape(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Convert string to index label using the StringIndexer\n",
    "# cat_col_list=['term','grade','sub_grade','home_ownership','verification_status','loan_status','pymnt_plan']\n",
    "# stringIndexer = feature.StringIndexer(inputCols=[cols for cols in cat_col_list],\n",
    "#                                       outputCols=[\"encoded_\" + cols for cols in cat_col_list])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets add the index column to be used later before we have to combine all the cleaned dataframes by index value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1 = part1.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# size_shape(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the VectorAssembler object\n",
    "# va = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"features\")\n",
    "# define the StandardScaler object\n",
    "# sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"features\", outputCol=\"scaled_features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 3 stages: stringindexer, VectorAssembler and StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1_pipe=Pipeline(stages=[stringIndexer, va, sScaler])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fitted_part1 = part1_pipe.fit(part1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part1 = fitted_part1.transform(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Pyspark to read the dataset and check missing values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "26-50 Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2=current_df1.select(current_df1.columns[25:50])\n",
    "# part2.columns\n",
    "# part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop date cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 22\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4341</td>\n",
       "      <td>10.3</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>113.98</td>\n",
       "      <td>53.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84.92</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16901</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12315</td>\n",
       "      <td>24.2</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>612.25</td>\n",
       "      <td>894.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>777.23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>1208</td>\n",
       "      <td>321915</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4599</td>\n",
       "      <td>19.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>353.89</td>\n",
       "      <td>353.89</td>\n",
       "      <td>212.79</td>\n",
       "      <td>141.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180.69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110299</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5468</td>\n",
       "      <td>78.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>286.71</td>\n",
       "      <td>286.71</td>\n",
       "      <td>168.07</td>\n",
       "      <td>118.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>146.51</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>686</td>\n",
       "      <td>305049</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>829</td>\n",
       "      <td>3.6</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>660.98</td>\n",
       "      <td>762.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>731.78</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>116007</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pub_rec revol_bal revol_util total_acc initial_list_status out_prncp out_prncp_inv total_pymnt total_pymnt_inv total_rec_prncp total_rec_int total_rec_late_fee recoveries collection_recovery_fee last_pymnt_amnt collections_12_mths_ex_med policy_code application_type acc_now_delinq tot_coll_amt tot_cur_bal open_acc_6m\n",
       "0  1       4341      10.3       34        w                   2386.02   2386.02       167.02      167.02          113.98          53.04         0.0                0.0        0.0                     84.92           0                          1           Individual       0              0            16901       2         \n",
       "1  1       12315     24.2       44        w                   29387.75  29387.75      1507.11     1507.11         612.25          894.86        0.0                0.0        0.0                     777.23          0                          1           Individual       0              1208         321915      4         \n",
       "2  0       4599      19.1       13        w                   4787.21   4787.21       353.89      353.89          212.79          141.10        0.0                0.0        0.0                     180.69          0                          1           Individual       0              0            110299      0         \n",
       "3  0       5468      78.1       13        w                   3831.93   3831.93       286.71      286.71          168.07          118.64        0.0                0.0        0.0                     146.51          0                          1           Individual       0              686          305049      1         \n",
       "4  0       829       3.6        26        w                   29339.02  29339.02      1423.21     1423.21         660.98          762.23        0.0                0.0        0.0                     731.78          0                          1           Individual       0              0            116007      3         "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the cols\n",
    "date_var=['last_pymnt_d','next_pymnt_d','last_credit_pull_d']\n",
    "part2=part2.drop(*date_var)\n",
    "print(size_shape(part2))\n",
    "part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec</td>\n",
       "      <td>1206975</td>\n",
       "      <td>0.1890756536458096</td>\n",
       "      <td>0.5624775092187297</td>\n",
       "      <td>PAYING DEBIT OFF.&lt;br/&gt;SLOW--BUT SURE.&lt;br/&gt; Borrower added on 06/02/10 &gt; JUST A MAN</td>\n",
       "      <td>debt_consolidation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_bal</td>\n",
       "      <td>1206977</td>\n",
       "      <td>16807.967260173642</td>\n",
       "      <td>22685.863720159345</td>\n",
       "      <td>LOOKING FOR A HAND-UP</td>\n",
       "      <td>To be rewarded with reasonable terms...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_util</td>\n",
       "      <td>1205863</td>\n",
       "      <td>49.61285204510645</td>\n",
       "      <td>25.884634838940773</td>\n",
       "      <td>and -- with the support of the members of the Lending Club -- we will prevail. Thank you for being a part of our future.&lt;br/&gt; Borrower added on 05/24/10 &gt; Here is some additional information about our company</td>\n",
       "      <td>Mar-1994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_acc</td>\n",
       "      <td>1206981</td>\n",
       "      <td>23.450838130462017</td>\n",
       "      <td>42.4613450933239</td>\n",
       "      <td>and the work we do.&lt;br/&gt;&lt;br/&gt;We are in the marketing research and consulting business. Our main task is to speak with customers</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>initial_list_status</td>\n",
       "      <td>1206980</td>\n",
       "      <td>926.0172727272726</td>\n",
       "      <td>3263.110236056832</td>\n",
       "      <td>IN ME. .&lt;br/&gt;\"</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp</td>\n",
       "      <td>1206978</td>\n",
       "      <td>8245.914858687509</td>\n",
       "      <td>8619.409939480662</td>\n",
       "      <td>but now I am completely back on track &amp; am doing whatever I can to raise my credit score back up</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp_inv</td>\n",
       "      <td>1206975</td>\n",
       "      <td>8244.153263444356</td>\n",
       "      <td>8618.754491613367</td>\n",
       "      <td>but in a nutshell we provide direct-from-the-customer data</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt</td>\n",
       "      <td>1206978</td>\n",
       "      <td>7857.123649318276</td>\n",
       "      <td>7372.222174135592</td>\n",
       "      <td>and opinions</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt_inv</td>\n",
       "      <td>1206981</td>\n",
       "      <td>7852.0274051305105</td>\n",
       "      <td>7369.046643175955</td>\n",
       "      <td>extremely rare to miss work.&lt;br/&gt;&lt;br/&gt;It's extremely late</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_prncp</td>\n",
       "      <td>1206980</td>\n",
       "      <td>5160.8152297105835</td>\n",
       "      <td>5195.473326471696</td>\n",
       "      <td>I need to hit the hay. If this loan even funds for the original amount (slightly lower APR) in comment that I was considering</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>1206982</td>\n",
       "      <td>2440.0997156951144</td>\n",
       "      <td>2747.4575185324143</td>\n",
       "      <td>of $2</td>\n",
       "      <td>9999.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_late_fee</td>\n",
       "      <td>1206981</td>\n",
       "      <td>1.9962355018726892</td>\n",
       "      <td>29.011852245782418</td>\n",
       "      <td>and continue to build upon them.&lt;br/&gt;\"</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>recoveries</td>\n",
       "      <td>1206981</td>\n",
       "      <td>254.5253475258523</td>\n",
       "      <td>977.1667932879081</td>\n",
       "      <td>0</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>1206981</td>\n",
       "      <td>42.2471526738202</td>\n",
       "      <td>171.24312905191488</td>\n",
       "      <td>0.0</td>\n",
       "      <td>xperienceddebtorqualifiedfor4xthisamount</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>1206982</td>\n",
       "      <td>489.2054205197111</td>\n",
       "      <td>700.1138703432373</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collections_12_mths_ex_med</td>\n",
       "      <td>1206968</td>\n",
       "      <td>0.037906233766018566</td>\n",
       "      <td>7.306349267798215</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>policy_code</td>\n",
       "      <td>1206975</td>\n",
       "      <td>1.0043449498414203</td>\n",
       "      <td>3.1299861759471175</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>application_type</td>\n",
       "      <td>1206971</td>\n",
       "      <td>1248.264375</td>\n",
       "      <td>4416.798379304432</td>\n",
       "      <td>0</td>\n",
       "      <td>Oct-2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_now_delinq</td>\n",
       "      <td>1206965</td>\n",
       "      <td>0.008077810776460108</td>\n",
       "      <td>4.894703732813887</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>1196681</td>\n",
       "      <td>216.85500005849536</td>\n",
       "      <td>1699.2516375435855</td>\n",
       "      <td>0</td>\n",
       "      <td>Oct-2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_cur_bal</td>\n",
       "      <td>1196684</td>\n",
       "      <td>138926.9121216863</td>\n",
       "      <td>159336.07953433666</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>1000683</td>\n",
       "      <td>0.9026380214214762</td>\n",
       "      <td>1.4790538763597334</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                     1                   2                                                                                                                                                                                                                  3                                         4\n",
       "summary                     count    mean                  stddev              min                                                                                                                                                                                                                max                                     \n",
       "pub_rec                     1206975  0.1890756536458096    0.5624775092187297   PAYING DEBIT OFF.<br/>SLOW--BUT SURE.<br/> Borrower added on 06/02/10 > JUST A MAN                                                                                                                                debt_consolidation                      \n",
       "revol_bal                   1206977  16807.967260173642    22685.863720159345   LOOKING FOR A HAND-UP                                                                                                                                                                                             To be rewarded with reasonable terms... \n",
       "revol_util                  1205863  49.61285204510645     25.884634838940773   and -- with the support of the members of the Lending Club -- we will prevail. Thank you for being a part of our future.<br/> Borrower added on 05/24/10 > Here is some additional information about our company  Mar-1994                                \n",
       "total_acc                   1206981  23.450838130462017    42.4613450933239     and the work we do.<br/><br/>We are in the marketing research and consulting business. Our main task is to speak with customers                                                                                   small_business                          \n",
       "initial_list_status         1206980  926.0172727272726     3263.110236056832    IN ME. .<br/>\"                                                                                                                                                                                                    w                                       \n",
       "out_prncp                   1206978  8245.914858687509     8619.409939480662    but now I am completely back on track & am doing whatever I can to raise my credit score back up                                                                                                                  f                                       \n",
       "out_prncp_inv               1206975  8244.153263444356     8618.754491613367    but in a nutshell we provide direct-from-the-customer data                                                                                                                                                        f                                       \n",
       "total_pymnt                 1206978  7857.123649318276     7372.222174135592    and opinions                                                                                                                                                                                                      w                                       \n",
       "total_pymnt_inv             1206981  7852.0274051305105    7369.046643175955    extremely rare to miss work.<br/><br/>It's extremely late                                                                                                                                                         f                                       \n",
       "total_rec_prncp             1206980  5160.8152297105835    5195.473326471696    I need to hit the hay. If this loan even funds for the original amount (slightly lower APR) in comment that I was considering                                                                                     f                                       \n",
       "total_rec_int               1206982  2440.0997156951144    2747.4575185324143   of $2                                                                                                                                                                                                             9999.70                                 \n",
       "total_rec_late_fee          1206981  1.9962355018726892    29.011852245782418   and continue to build upon them.<br/>\"                                                                                                                                                                            f                                       \n",
       "recoveries                  1206981  254.5253475258523     977.1667932879081   0                                                                                                                                                                                                                  small_business                          \n",
       "collection_recovery_fee     1206981  42.2471526738202      171.24312905191488  0.0                                                                                                                                                                                                                xperienceddebtorqualifiedfor4xthisamount\n",
       "last_pymnt_amnt             1206982  489.2054205197111     700.1138703432373   0.0                                                                                                                                                                                                                f                                       \n",
       "collections_12_mths_ex_med  1206968  0.037906233766018566  7.306349267798215   0                                                                                                                                                                                                                  f                                       \n",
       "policy_code                 1206975  1.0043449498414203    3.1299861759471175  0                                                                                                                                                                                                                  f                                       \n",
       "application_type            1206971  1248.264375           4416.798379304432   0                                                                                                                                                                                                                  Oct-2016                                \n",
       "acc_now_delinq              1206965  0.008077810776460108  4.894703732813887   0                                                                                                                                                                                                                  Individual                              \n",
       "tot_coll_amt                1196681  216.85500005849536    1699.2516375435855  0                                                                                                                                                                                                                  Oct-2016                                \n",
       "tot_cur_bal                 1196684  138926.9121216863     159336.07953433666  0                                                                                                                                                                                                                  f                                       \n",
       "open_acc_6m                 1000683  0.9026380214214762    1.4790538763597334  0                                                                                                                                                                                                                  Individual                              "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the summary of each col\n",
    "part2.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>1120</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>10302</td>\n",
       "      <td>10299</td>\n",
       "      <td>206300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  policy_code  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  8        6          1120        2          3                    5          8              5            2                3                1              2                   2           2                        1                15                          8            12                18              10302         10299        206300     "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the missing values\n",
    "checkna_part2 = part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns])\n",
    "checkna_part2.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cope with 3 categorical variables: 'initial_list_status', 'application_type', 'policy_code'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- initial_list_status: string (nullable = true)\n",
      " |-- application_type: string (nullable = true)\n",
      " |-- policy_code: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cat_col=['initial_list_status', 'application_type','policy_code']\n",
    "part2_cat_col=part2.select(cat_col)\n",
    "part2_cat_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2=part2.filter(\"initial_list_status in ('w','f')\")\n",
    "part2=part2.filter(\"application_type in ('Individual','Joint App')\")\n",
    "part2=part2.filter(\"policy_code in ('1','2')\")\n",
    "# part2.select(fn.countDistinct(\"initial_list_status\")).show() returns 2\n",
    "# part2.select(\"application_type\").distinct().show() returns 'Joint App' and 'Individual'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# part2.groupBy(\"application_type\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.groupBy(\"initial_list_status\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-------+\n",
      "|policy_code|  count|\n",
      "+-----------+-------+\n",
      "|          1|1206949|\n",
      "+-----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.groupBy(\"policy_code\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this case, the variable 'policy_code' should be dropped because all the samples belong to one category, which doesn't provide any useful information for classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# current_df.groupBy(\"policy_code\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2=part2.drop(\"policy_code\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cope with numerical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2_num_col=part2.drop(*cat_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# part2_num_col.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with 'pub_rec' variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+\n",
      "|count(DISTINCT pub_rec)|\n",
      "+-----------------------+\n",
      "|                     35|\n",
      "+-----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# how many distinct values in pub_rec feature\n",
    "part2.select(fn.countDistinct(\"pub_rec\")).show()\n",
    "# number of samples that have pub_rec\n",
    "# part2.filter(fn.col('pub_rec')!=0).count() returns 185562"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the distinct values \n",
    "# part2.select('pub_rec').distinct().show(52)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # REPLACE 11.59 15.31 etc\n",
    "# # [0-9]+\\.[0-9]+\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"[0-9]+\\.[0-9]+\", \"null\"))\n",
    "\n",
    "# # remove strings\n",
    "# # ^[\\D]+.*$\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"^[\\D]+.*$\", \"null\"))\n",
    "\n",
    "# # remove 120XX\n",
    "# # (^.*[\\D]+$)\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"(^.*[\\D]+$)\", \"null\"))\n",
    "\n",
    "# part2.select(fn.countDistinct(\"pub_rec\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+\n",
      "|summary|            pub_rec|\n",
      "+-------+-------------------+\n",
      "|  count|            1206949|\n",
      "|   mean|0.18891601882101067|\n",
      "| stddev| 0.5552736651970246|\n",
      "|    min|                  0|\n",
      "|    max|                  9|\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('pub_rec').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|         revol_bal|\n",
      "+-------+------------------+\n",
      "|  count|           1206949|\n",
      "|   mean|16808.217865046492|\n",
      "| stddev| 22685.94006979832|\n",
      "|    min|                 0|\n",
      "|    max|             99996|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('revol_bal').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We conver the data type of each col to integer or float"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.withColumn('pub_rec', part2['pub_rec'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('revol_bal', part2['revol_bal'].cast(FloatType()))\n",
    "part2 = part2.withColumn('revol_util',part2['revol_util'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_acc', part2['total_acc'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('out_prncp', part2['out_prncp'].cast(FloatType()))\n",
    "part2 = part2.withColumn('out_prncp_inv', part2['out_prncp_inv'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_pymnt', part2['total_pymnt'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_pymnt_inv', part2['total_pymnt_inv'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_prncp', part2['total_rec_prncp'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_int', part2['total_rec_int'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_late_fee', part2['total_rec_late_fee'].cast(FloatType()))\n",
    "part2 = part2.withColumn('recoveries', part2['recoveries'].cast(FloatType()))\n",
    "part2 = part2.withColumn('collection_recovery_fee', part2['collection_recovery_fee'].cast(FloatType()))\n",
    "part2 = part2.withColumn('last_pymnt_amnt', part2['last_pymnt_amnt'].cast(FloatType()))\n",
    "part2 = part2.withColumn('collections_12_mths_ex_med', part2['collections_12_mths_ex_med'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('acc_now_delinq', part2['acc_now_delinq'].cast(IntegerType()))\n",
    "part2 = part2.withColumn(\"open_acc_6m\",part2[\"open_acc_6m\"].cast(IntegerType()))\n",
    "part2 = part2.withColumn(\"tot_coll_amt\",part2[\"tot_coll_amt\"].cast(FloatType()))\n",
    "part2 = part2.withColumn(\"tot_cur_bal\",part2[\"tot_cur_bal\"].cast(FloatType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- pub_rec: integer (nullable = true)\n",
      " |-- revol_bal: float (nullable = true)\n",
      " |-- revol_util: float (nullable = true)\n",
      " |-- total_acc: integer (nullable = true)\n",
      " |-- initial_list_status: string (nullable = true)\n",
      " |-- out_prncp: float (nullable = true)\n",
      " |-- out_prncp_inv: float (nullable = true)\n",
      " |-- total_pymnt: float (nullable = true)\n",
      " |-- total_pymnt_inv: float (nullable = true)\n",
      " |-- total_rec_prncp: float (nullable = true)\n",
      " |-- total_rec_int: float (nullable = true)\n",
      " |-- total_rec_late_fee: float (nullable = true)\n",
      " |-- recoveries: float (nullable = true)\n",
      " |-- collection_recovery_fee: float (nullable = true)\n",
      " |-- last_pymnt_amnt: float (nullable = true)\n",
      " |-- collections_12_mths_ex_med: integer (nullable = true)\n",
      " |-- application_type: string (nullable = true)\n",
      " |-- acc_now_delinq: integer (nullable = true)\n",
      " |-- tot_coll_amt: float (nullable = true)\n",
      " |-- tot_cur_bal: float (nullable = true)\n",
      " |-- open_acc_6m: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10285</td>\n",
       "      <td>10285</td>\n",
       "      <td>206280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0        0          1115        0          0                    0          0              0            0                0                0              0                   0           0                        0                6                           0                 0               10285         10285        206280     "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's cope with the columns with missing values:revol_util, collections_12_mths_ex_med, tot_coll_amt, tot_cur_bal, open_acc_6m <br>\n",
    "By replacing the missing values with the mean values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+\n",
      "|summary|       revol_util|\n",
      "+-------+-----------------+\n",
      "|  count|          1205834|\n",
      "|   mean|49.60215702112277|\n",
      "| stddev|24.76317095036885|\n",
      "|    min|              0.0|\n",
      "|    max|            366.6|\n",
      "+-------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('revol_util').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49.60215702112277"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "revol_util_mean=part2.select(fn.avg('revol_util')).collect()\n",
    "revol_util_mean[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.na.fill({'revol_util':revol_util_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------------------------------------------------------+\n",
      "|count(CASE WHEN (isnan(revol_util) OR (revol_util IS NULL)) THEN true END)|\n",
      "+--------------------------------------------------------------------------+\n",
      "|                                                                         0|\n",
      "+--------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# part2.select([count(when(isnan('revol_util'),True))]).show()\n",
    "# part2.select([count(when(col('revol_util').isNull(),True))]).show()\n",
    "part2.select([count(when(isnan('revol_util') | col('revol_util').isNull() , True))]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEICAYAAACqMQjAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAej0lEQVR4nO3de5hdVX3/8ffHBMKdABkpJIEESauResGRxJ83SnwgoWiwD9KAlUhjUyl49ylBrCCiVVuhpSIWTSRcQsB4IbVQTLkU/f0gONwJETPlkowBMhgCQRQMfH9/rDWwc3LWTGbO5JxJ8nk9z3lmn+9ee+/vWeec/T1r733OKCIwMzOr51WtTsDMzIYuFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpFoMUnflvQPg7SuAyQ9K2lYvn+zpI8Mxrrz+q6TNHOw1teP7Z4r6UlJjzd72yWSDpfUtZltt0i/1eYgaZmkwwd7O/0l6Z2SHqzcf0TSe/L05yR9dwtsc9DeR7ax4a1OYFsm6RFgX2AD8CLwAHApcHFEvAQQER/tx7o+EhH/XWoTESuB3RrL+uXtnQ0cHBF/VVn/tMFYdz/zGAt8BjgwItbUmX84cCPwHBDAauCrEfG9ZubZm2b1W0S8vhnbqSUpgAkR0Znz+BnwJ/XaRsRXBmF7Hya9F95RWe9mvY+aobY/tnYeSWx5742I3YEDga8CpwNzB3sjkrbVgn8g8Jt6BaJidUTsBuwBfAr4jqS6OymzRmzD77OyiPBtC92AR4D31MQOA14CDsn3LwHOzdOjgJ8A64C1wM9IhfyyvMzvgGeBvwfGkT45zwJWArdUYsPz+m4G/hG4HXgauAbYO887HOiqly8wFXgB+EPe3j2V9X0kT78K+DzwKLCGNELaM8/ryWNmzu1J4Mxe+mnPvHx3Xt/n8/rfkx/zSzmPS+osW+9xrAE+ULn/WmBJ7tMHgeNzfDLwODCs0vb9wL15egTwL6TRyeo8PaJ2u8AcYFFNDv8KXFCn3z4M/Bz4Z+Ap4GFgWmW58fm5XA/8N3AhcHmh3zZ67FReb8DZwNW5X9cDy4D2Stv9gR/kPn8Y+Hgvz8/L+VcfQ56+JT/Xv83P0V9uRl6X5+lv5mV6bhuAsyt9+r859weA9+f464Dfk0bmzwLrat9H+f7fAJ35OV8M7F+ZF8BHgRX5ObgQUOGxnw0sAi4HngE+QnoP30p6nz6WH8eOpf7I8WOAu/My/w94Q6v3T5t780iiySLidqALeGed2Z/J89pIh6k+lxaJD5F2tu+NiN0i4uuVZd5NeuMcVdjkScBfk3YKG4ALNiPH/wK+AlyVt/fGOs0+nG9/BhxEOsz1zZo27yAddpgCfEHS6wqb/DdSoTgoP56TgJMjHVqbRh4pRMSHe8tb0qskvY9UbDtzbFdSgVgAvBo4AfiWpNdHxG2kN/MRldWcmNsCnEkqJG8C3kjaOXy+zqavBI6WtEfe5jDg+Mp6ak0iFatRwNeBuZKU5y0gFfV9SDuoD/X2mPvwPmAhMJK0o/xmzu9VwH8A9wCjSc/PJyWVXkNFEfGuPPnG/Bxd1Y9lT8vL7EZ6rTxF+iADqUC8k/S6+CJwuaT9ImI5aQd/a152ZO16JR1B+nB0PLAf6YPHwppmxwBvJT2vx1N+/wBMJxWKkcAVpAL1KdLz9zZS//1dfkyb9IekQ4F5wN+Sntd/BxZLGtF3L7Wei0RrrAb2rhP/A+lFfWBE/CEifhb5Y0gvzo6I30bE7wrzL4uI+yPit8A/AMf3nNhu0AeB8yLioYh4FjgDmFEzHP9iRPwuIu4h7ZA2KTY5l78EzoiI9RHxCPAN+rdz3F/SOtKo40fApyPirjzvGOCRiPheRGyIiDtJn6CPy/OvJBUOJO0OHJ1jPY/xnIhYExHdpJ3VJnlFxKPAncCxOXQE8FwuQvU8GhHfiYgXgfmk53xfSQeQdlxfiIgXIuLnpJ37QP08Iq7N27mMV/r/rUBbRJyTt/MQ8B1gRgPbGjBJbcCPgY/1PG8R8f2IWB0RL+XCs4JUpDfHB4F5EXFnRDxPem2+TdK4SpuvRsS6SOfxbiJ9ECi5NSJ+nHP5XUTcERG35dfTI6Sd/rt7Wf5vgH+PiKUR8WJEzAeeJ30AGfJcJFpjNGkYXOufSJ+AfyrpIUlzNmNdq/ox/1FgB9InoEbtn9dXXfdw0gioR/VqpOeof1J9FLBjnXWN7kcuq/Mnyj1II6XqyOBAYJKkdT030k7kj/L8BcBf5E91fwHcmXf6UP8x7l/IYQG52LDxaKSel/slIp7Lk7vlda+txKDv57c3tf2/Uy7iB5ILa6VPPsfGz11TSNqB9Cl9QUQsrMRPknR3Jb9D2PzX7UbPW/4Q8xs2fk1tzmuzx0bPgaQ/lvQTSY9LeoY06u4ttwOBz9T091jKr6UhxUWiySS9lfRi/XntvPxJ+jMRcRDwXuDTkqb0zC6ssq+RxtjK9AGk0cqTpMMsu1TyGkY6zLW5611NevFX170BeKKP5Wo9mXOqXdev+7ke8qfG04E/ldTzqX4V8D8RMbJy2y0iTsnLPEDaoUxj0517vce4urD57wOHSxpDOq/RW5EoeQzYW9IuldjYUuMGrAIerumT3SPi6EL7jV4rvFJgB8O/kc47vHwYT9KBpJHNacA++QPA/UDPIbl+vTbzIcd9GMBrqrC9i4Bfkq5g2oNUYLXJUq9YBXy5pr93iYgre1lmyHCRaBJJe0g6hnRs9PKIuK9Om2MkHZyPTz9DOvb5Yp79BOmYfX/9laSJecdzDukE64vAr0ifLP88f5r7POlEbY8ngHH5+HU9VwKfkjRe0m68cg5jQ3+Sy7lcDXxZ0u55B/Fp0onCfouIF0iHq76QQz8B/ljShyTtkG9vrTk/sgD4OPAu0s6++hg/L6lN0qi8zrp55cNRNwPfI+2Alw8g90eBDuBsSTtKehvpw8Jgux14RtLpknaWNEzSIfkDTD13k0Zbu0g6mHSxRNWAXpuS/pZ0mObEyJeEZ7uSdszdud3JpJFEdXtjJO1YWPUC4GRJb8ojxK8AS/OhocGwO+n9+ayk1wKn1Myv7Y/vAB+VNEnJrvl9t/sg5bNFuUhsef8haT3p08SZwHnAyYW2E0hXtDxLunriWxFxc573j6Qd1jpJn+3H9i8jXfnxOLATaWdIRDxNOtn2XdInrN+STpr36NlZ/kbSnXXWOy+v+xbS1TG/Bz7Wj7yqPpa3/xBphLUgr3+g5gEHSHpvRKwHjiQdb19N6oevsXFBvJJ0Rc6NEfFkJX4uaad9L3Af6bzDub1sdwHpiqyBjCJ6fJB0MvQ3eVtXkY5fD5pcmN9LOg7/MGk0913SSeJ6zidd7fYE6RzKFTXzzwbm59fm8f1I5QTSznS10pdAn5X0uTy6+wbpPfAE8KfA/60sdyPpaq3HJT1Zu9KIuIF0/u0HpNHZaxjc8y2fJY0615MKQO3J+rOp9EdEdJDOS3yTdHK+k3TRx1ZBfZ8XNbNWkXQV8MuIOKvVudj2ySMJsyEkHwp7Tb6cdyrp8ssftzov235tf98eNBva/gj4IelEaxdwSuVyXrOm8+EmMzMr8uEmMzMr2uYON40aNSrGjRvX6jTMzLYqd9xxx5MR0VYb3+aKxLhx4+jo6Gh1GmZmWxVJj9aL+3CTmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRVtc9+43tYsWLqybvzESQc0ORMz2x55JGFmZkUuEmZmVtRnkZA0T9IaSfdXYv8k6ZeS7pX0I0kjK/POkNQp6UFJR1XiU3OsU9KcSny8pKWSVki6quefm0sake935vnjButBm5nZ5tmckcQlwNSa2BLgkIh4A/Ar4AwASRNJ/3D89XmZb0kaJmkYcCEwDZgInJDbQvqn9OdHxATSPwmfleOzgKci4mDSP2L/2oAeoZmZDVifRSIibgHW1sR+GhEb8t3bgDF5ejqwMCKej4iHgU7gsHzrjIiHIuIFYCEwXZKAI4BFefn5wLGVdc3P04uAKbm9mZk1yWCck/hr4Lo8PRpYVZnXlWOl+D7AukrB6YlvtK48/+ncfhOSZkvqkNTR3d3d8AMyM7OkoSIh6UxgA3BFT6hOsxhAvLd1bRqMuDgi2iOiva1tk3+sZGZmAzTg70lImgkcA0yJiJ6ddxcwttJsDLA6T9eLPwmMlDQ8jxaq7XvW1SVpOLAnNYe9tiWl70OYmbXSgIqEpKnA6cC7I+K5yqzFwAJJ5wH7AxOA20mjggmSxgO/Jp3cPjEiQtJNwHGk8xQzgWsq65oJ3Jrn31gpRts9f8nOzJqhzyIh6UrgcGCUpC7gLNLVTCOAJflc8m0R8dGIWCbpauAB0mGoUyPixbye04DrgWHAvIhYljdxOrBQ0rnAXcDcHJ8LXCapkzSCmDEIj9fMzPpB29qH8/b29ujo6Gh1Gv02WIebPJIws4GQdEdEtNfG/Y1rMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMr8r8vbTL//IaZbU08kjAzsyIXCTMzK/Lhpm2Mf/jPzAaTRxJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkV9FglJ8yStkXR/Jba3pCWSVuS/e+W4JF0gqVPSvZIOrSwzM7dfIWlmJf4WSfflZS6QpN62YWZmzbM5I4lLgKk1sTnADRExAbgh3weYBkzIt9nARZB2+MBZwCTgMOCsyk7/oty2Z7mpfWzDBmDB0pV1b2ZmvemzSETELcDamvB0YH6eng8cW4lfGsltwEhJ+wFHAUsiYm1EPAUsAabmeXtExK0REcClNeuqtw0zM2uSgZ6T2DciHgPIf1+d46OBVZV2XTnWW7yrTry3bWxC0mxJHZI6uru7B/iQzMys1mCfuFadWAwg3i8RcXFEtEdEe1tbW38XNzOzgoH+j+snJO0XEY/lQ0ZrcrwLGFtpNwZYneOH18RvzvExddr3to0hz8f6zWxbMdCRxGKg5wqlmcA1lfhJ+SqnycDT+VDR9cCRkvbKJ6yPBK7P89ZLmpyvajqpZl31tmFmZk3S50hC0pWkUcAoSV2kq5S+ClwtaRawEvhAbn4tcDTQCTwHnAwQEWslfQn4RW53TkT0nAw/hXQF1c7AdflGL9swM7Mm6bNIRMQJhVlT6rQN4NTCeuYB8+rEO4BD6sR/U28bZmbWPP7GtZmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWVGf/77Utm0Llq6sGz9x0gFNzsTMhiKPJMzMrMhFwszMilwkzMysyEXCzMyKXCTMzKyooSIh6VOSlkm6X9KVknaSNF7SUkkrJF0lacfcdkS+35nnj6us54wcf1DSUZX41BzrlDSnkVzNzKz/BlwkJI0GPg60R8QhwDBgBvA14PyImAA8BczKi8wCnoqIg4HzczskTczLvR6YCnxL0jBJw4ALgWnAROCE3NbMzJqk0cNNw4GdJQ0HdgEeA44AFuX584Fj8/T0fJ88f4ok5fjCiHg+Ih4GOoHD8q0zIh6KiBeAhbmtmZk1yYCLRET8GvhnYCWpODwN3AGsi4gNuVkXMDpPjwZW5WU35Pb7VOM1y5Tim5A0W1KHpI7u7u6BPiQzM6vRyOGmvUif7McD+wO7kg4N1YqeRQrz+hvfNBhxcUS0R0R7W1tbX6mbmdlmauRw03uAhyOiOyL+APwQ+D/AyHz4CWAMsDpPdwFjAfL8PYG11XjNMqW4mZk1SSO/3bQSmCxpF+B3wBSgA7gJOI50DmEmcE1uvzjfvzXPvzEiQtJiYIGk80gjkgnA7aSRxARJ44Ffk05un9hAvoOu9LtHZmbbigEXiYhYKmkRcCewAbgLuBj4T2ChpHNzbG5eZC5wmaRO0ghiRl7PMklXAw/k9ZwaES8CSDoNuJ505dS8iFg20HzNzKz/FFH3MP9Wq729PTo6OpqyrW15JOFfgTXbvki6IyLaa+P+xrWZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVlRIz8Vbtuw0o8X+of/zLYvHkmYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZUUNFQtJISYsk/VLScklvk7S3pCWSVuS/e+W2knSBpE5J90o6tLKembn9CkkzK/G3SLovL3OBJDWSr5mZ9U+jI4l/Bf4rIl4LvBFYDswBboiICcAN+T7ANGBCvs0GLgKQtDdwFjAJOAw4q6ew5DazK8tNbTBfMzPrhwEXCUl7AO8C5gJExAsRsQ6YDszPzeYDx+bp6cClkdwGjJS0H3AUsCQi1kbEU8ASYGqet0dE3BoRAVxaWZeZmTVBIyOJg4Bu4HuS7pL0XUm7AvtGxGMA+e+rc/vRwKrK8l051lu8q07czMyapJEiMRw4FLgoIt4M/JZXDi3VU+98QgwgvumKpdmSOiR1dHd39561mZlttkaKRBfQFRFL8/1FpKLxRD5URP67ptJ+bGX5McDqPuJj6sQ3EREXR0R7RLS3tbU18JDMzKxqwEUiIh4HVkn6kxyaAjwALAZ6rlCaCVyTpxcDJ+WrnCYDT+fDUdcDR0raK5+wPhK4Ps9bL2lyvqrppMq6zMysCRr996UfA66QtCPwEHAyqfBcLWkWsBL4QG57LXA00Ak8l9sSEWslfQn4RW53TkSszdOnAJcAOwPX5ZuZmTVJQ0UiIu4G2uvMmlKnbQCnFtYzD5hXJ94BHNJIjoOh9P+ezcy2df7GtZmZFblImJlZkYuEmZkVuUiYmVlRo1c32XamdBL/xEkHNDkTM2sGjyTMzKzIRcLMzIpcJMzMrMhFwszMilwkzMysyEXCzMyKXCTMzKzIRcLMzIpcJMzMrMjfuK7wT4KbmW3MIwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzooaLhKRhku6S9JN8f7ykpZJWSLpK0o45PiLf78zzx1XWcUaOPyjpqEp8ao51SprTaK5mZtY/gzGS+ASwvHL/a8D5ETEBeAqYleOzgKci4mDg/NwOSROBGcDrganAt3LhGQZcCEwDJgIn5LZmZtYkDRUJSWOAPwe+m+8LOAJYlJvMB47N09PzffL8Kbn9dGBhRDwfEQ8DncBh+dYZEQ9FxAvAwtzWzMyapNGRxL8Afw+8lO/vA6yLiA35fhcwOk+PBlYB5PlP5/Yvx2uWKcU3IWm2pA5JHd3d3Q0+JDMz6zHgIiHpGGBNRNxRDddpGn3M629802DExRHRHhHtbW1tvWRtZmb90cj/k3g78D5JRwM7AXuQRhYjJQ3Po4UxwOrcvgsYC3RJGg7sCaytxHtUlynFzcysCQY8koiIMyJiTESMI514vjEiPgjcBByXm80ErsnTi/N98vwbIyJyfEa++mk8MAG4HfgFMCFfLbVj3sbigeZrZmb9tyX+M93pwEJJ5wJ3AXNzfC5wmaRO0ghiBkBELJN0NfAAsAE4NSJeBJB0GnA9MAyYFxHLtkC+ZmZWMChFIiJuBm7O0w+RrkyqbfN74AOF5b8MfLlO/Frg2sHI0czM+s/fuDYzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzoi3xT4dsO7Rg6cq68RMnHdDkTMxsMHkkYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZWNOAiIWmspJskLZe0TNIncnxvSUskrch/98pxSbpAUqekeyUdWlnXzNx+haSZlfhbJN2Xl7lAkhp5sGZm1j+NjCQ2AJ+JiNcBk4FTJU0E5gA3RMQE4IZ8H2AaMCHfZgMXQSoqwFnAJOAw4KyewpLbzK4sN7WBfM3MrJ8GXCQi4rGIuDNPrweWA6OB6cD83Gw+cGyeng5cGsltwEhJ+wFHAUsiYm1EPAUsAabmeXtExK0REcCllXWZmVkTDMo5CUnjgDcDS4F9I+IxSIUEeHVuNhpYVVmsK8d6i3fVidfb/mxJHZI6uru7G304ZmaWNVwkJO0G/AD4ZEQ801vTOrEYQHzTYMTFEdEeEe1tbW19pWxmZpupoSIhaQdSgbgiIn6Yw0/kQ0Xkv2tyvAsYW1l8DLC6j/iYOnEzM2uSRq5uEjAXWB4R51VmLQZ6rlCaCVxTiZ+Ur3KaDDydD0ddDxwpaa98wvpI4Po8b72kyXlbJ1XWZWZmTdDI/5N4O/Ah4D5Jd+fY54CvAldLmgWsBD6Q510LHA10As8BJwNExFpJXwJ+kdudExFr8/QpwCXAzsB1+WZmZk0y4CIRET+n/nkDgCl12gdwamFd84B5deIdwCEDzdHMzBrjb1ybmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVnRkC8SkqZKelBSp6Q5rc7HzGx7MqSLhKRhwIXANGAicIKkia3Nysxs+zG81Qn04TCgMyIeApC0EJgOPNDSrGyzLVi6sm78xEkHNDkTMxuIoV4kRgOrKve7gEm1jSTNBmbnu89KenCA2xsFPDnAZZtpa8iz1xw/2MRE+rDV9+UQsjXkuTXkCK3J88B6waFeJFQnFpsEIi4GLm54Y1JHRLQ3up4tbWvIc2vIEbaOPLeGHGHryHNryBGGVp5D+pwEaeQwtnJ/DLC6RbmYmW13hnqR+AUwQdJ4STsCM4DFLc7JzGy7MaQPN0XEBkmnAdcDw4B5EbFsC26y4UNWTbI15Lk15AhbR55bQ46wdeS5NeQIQyhPRWxyiN/MzAwY+oebzMyshVwkzMysyEUiG6o//yHpEUn3SbpbUkeO7S1piaQV+e9eLchrnqQ1ku6vxOrmpeSC3Lf3Sjq0hTmeLenXuT/vlnR0Zd4ZOccHJR3VjBzzdsdKuknScknLJH0ix4dMf/aS45DqT0k7Sbpd0j05zy/m+HhJS3NfXpUvhEHSiHy/M88f18IcL5H0cKUv35TjLXn/vCwitvsb6aT4/wIHATsC9wATW51Xzu0RYFRN7OvAnDw9B/haC/J6F3AocH9feQFHA9eRvvcyGVjawhzPBj5bp+3E/LyPAMbn18OwJuW5H3Bont4d+FXOZ8j0Zy85Dqn+zH2yW57eAVia++hqYEaOfxs4JU//HfDtPD0DuKqFOV4CHFenfUvePz03jySSl3/+IyJeAHp+/mOomg7Mz9PzgWObnUBE3AKsrQmX8poOXBrJbcBISfu1KMeS6cDCiHg+Ih4GOkmviy0uIh6LiDvz9HpgOenXBoZMf/aSY0lL+jP3ybP57g75FsARwKIcr+3Lnj5eBEyRVO9LvM3IsaQl758eLhJJvZ//6O0N0EwB/FTSHfnnRwD2jYjHIL15gVe3LLuNlfIaav17Wh62z6scqhsSOebDHW8mfbockv1ZkyMMsf6UNEzS3cAaYAlpFLMuIjbUyeXlPPP8p4F9mp1jRPT05ZdzX54vaURtjnXy3+JcJJLN+vmPFnl7RBxK+iXcUyW9q9UJDcBQ6t+LgNcAbwIeA76R4y3PUdJuwA+AT0bEM701rRNrSq51chxy/RkRL0bEm0i/0HAY8LpecmlJnrU5SjoEOAN4LfBWYG/g9Fbm2MNFIhmyP/8REavz3zXAj0gv+id6hpv575rWZbiRUl5Dpn8j4on8Bn0J+A6vHAJpaY6SdiDtfK+IiB/m8JDqz3o5DtX+zLmtA24mHccfKanny8PVXF7OM8/fk80/RDmYOU7Nh/QiIp4HvscQ6UsXiWRI/vyHpF0l7d4zDRwJ3E/KbWZuNhO4pjUZbqKU12LgpHyVxmTg6Z7DKM1Wcyz3/aT+hJTjjHy1y3hgAnB7k3ISMBdYHhHnVWYNmf4s5TjU+lNSm6SReXpn4D2k8yc3AcflZrV92dPHxwE3Rj5b3OQcf1n5QCDSOZNqX7bu/dPMs+RD+Ua6guBXpOOXZ7Y6n5zTQaQrRO4BlvXkRTpmegOwIv/duwW5XUk6vPAH0iedWaW8SMPlC3Pf3ge0tzDHy3IO95LefPtV2p+Zc3wQmNbEvnwH6fDBvcDd+Xb0UOrPXnIcUv0JvAG4K+dzP/CFHD+IVKQ6ge8DI3J8p3y/M88/qIU53pj78n7gcl65Aqol75+em3+Ww8zMiny4yczMilwkzMysyEXCzMyKXCTMzKzIRcLMzIpcJMzMrMhFwszMiv4/Ou0WMz6ncYcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(part2.select('revol_util').collect(),kde=False).set_title('Distribution of Revolving line utilization rate');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.select('revol_util').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEICAYAAACTVrmbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdAklEQVR4nO3df5QV9Znn8fcngEIUww87DgEVHDFKiDGkFVyzGRKCto5JO+dg0LgRMzisETPGnd0BzRxlk3g0sxmZ8cQko4GAxAENyUQSYZj213jmrCCtEhUQ6VWjLUQQ/P0bffaP+l5y7b7d3G7q3tvdfF7n3NNVT32r6qnS62N963urFBGYmZnl6UO1TsDMzPoeFxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMxqRNLTkr64j9u4QNJ/5pWTWV5cXMzMLHcuLmY1IGkJcATwG0mvSfpbSZMk/V9JL0n6naTJRe0vkPSkpFclPSXpPEnHAT8BTk7beKlGh2PWjvz4F7PakPQ0cGFE3ClpJPAI8DXg34ApwDLgWOANYBtwYkRsljQCGBYRGyRdkLbx2Vocg1lHfOVi1jP8N2BlRKyMiPcjogloBs5Iy98HxksaFBHbImJDzTI1K4OLi1nPcCRwduoSeyl1cX0WGBERrwPTgYuAbZLukHRsLZM12xsXF7PaKe6TfhZYEhFDij4HRcS1ABGxOiKmAiOAx4GbSmzDrMdwcTGrneeBo9L0z4EvSTpNUj9JAyVNljRK0mGSvizpIOBt4DXgvaJtjJJ0QPXTN+uYi4tZ7VwD/F3qApsONAJXADvIrmT+F9l39EPA3wBbgV3AnwEXp23cDWwA/iDphapmb9YJjxYzM7Pc+crFzMxy5+JiZma5c3ExM7PcubiYmVnu+tc6gZ7i0EMPjdGjR9c6DTOzXuXBBx98ISLq2sZdXJLRo0fT3Nxc6zTMzHoVSb8vFXe3mJmZ5c7FxczMcufiYmZmufM9FzPr8959911aW1t56623ap1KrzVw4EBGjRrFgAEDymrv4mJmfV5rayuDBw9m9OjRSKp1Or1ORLBz505aW1sZM2ZMWeu4W8zM+ry33nqL4cOHu7B0kySGDx/epSs/Fxcz2y+4sOybrp4/FxczM8ud77mY2X5nftMTuW7vsqnH5Lq9vsBXLn3U/KYn2n3MrO8ZPXo0L7zQ9ffELVq0iK1bt+6Zv/DCC9m4ceM+bbOYi4uZWZVFBO+//35Nc2hbXH76058ybty43Lbv4mJmVgVPP/00xx13HBdffDETJkxgyZIlnHzyyUyYMIGzzz6b1157jVWrVvGVr3xlzzr33nsvX/rSlwBYunQpn/zkJxk/fjxz5swpe5/jx4/fM/+DH/yAefPmsXz5cpqbmznvvPM44YQTePPNN5k8eXKuz1esWHGRtFDSdkmPFcX+j6THJT0i6V8lDSladrmkFkmbJZ1WFG9IsRZJc4viYyStlbRF0q2SDkjxA9N8S1o+ulLHaGbWFZs3b+b888+nqamJBQsWcOedd/LQQw9RX1/Pddddx9SpU1mzZg2vv/46ALfeeivTp09n69atzJkzh7vvvpv169ezbt06fv3rX3c7j2nTplFfX88tt9zC+vXrGTRoUF6HuEclr1wWAQ1tYk3A+Ig4HngCuBxA0jjgHOATaZ0fSeonqR9wA3A6MA44N7UF+D4wPyLGAi8CM1N8JvBiRBwNzE/tzMxq7sgjj2TSpEmsWbOGjRs3csopp3DCCSewePFifv/739O/f38aGhr4zW9+w+7du7njjjtobGxk3bp1TJ48mbq6Ovr37895553HfffdV+vD6VTFRotFxH1trxoi4t+LZtcA09J0I7AsIt4GnpLUApyUlrVExJMAkpYBjZI2AV8AvpraLAbmAT9O25qX4suBH0pSRERuB2dm1g0HHXQQkN1zmTp1KkuXLm3XZvr06dxwww0MGzaME088kcGDB9Pd/3z179//A/d2qvn4m1oORf5L4NY0PZKs2BS0phjAs23iE4HhwEsRsbtE+5GFdSJit6SXU/t2Qx8kzQJmARxxxBH7eDhm1lvUeujwpEmTmD17Ni0tLRx99NG88cYbtLa2cswxxzB58mRmzpzJTTfdxPTp0wGYOHEil156KS+88AJDhw5l6dKlfPOb39zrfg477DC2b9/Ozp07Ofjgg/ntb39LQ0PWoTR48GBeffXVih1jTW7oS/o2sBu4pRAq0Sy6Ee9sW+2DETdGRH1E1NfVtXuRmplZRdTV1bFo0SLOPfdcjj/+eCZNmsTjjz8OQL9+/TjzzDNZtWoVZ555JgAjRozgmmuu4fOf/zyf+tSnmDBhAo2NjXvdz4ABA7jyyiuZOHEiZ555Jscee+yeZRdccAEXXXTRnhv6eVMle4tSt9hvI2J8UWwGcBEwJSLeSLHLASLimjS/mj92bc2LiNOK2wHXAjuAP0lXJycX2hXWjYj7JfUH/gDU7a1brL6+PnrjmyhL/X7lsqnHdBg32x9t2rSJ4447rtZp9HqlzqOkByOivm3bqnaLSWoA5gB/VigsyQrgXyRdB3wMGAs8QHYVMlbSGOA5spv+X42IkHQP2T2bZcAM4Paibc0A7k/L7/b9lj9y0TGzaqhYcZG0FJgMHCqpFbiKbHTYgUBTegjamoi4KCI2SLoN2EjWXTY7It5L27kEWA30AxZGxIa0iznAMknfAx4GFqT4AmBJGhSwi6wgmZn1WTt37mTKlCnt4nfddRfDhw+vQUaVHS12bonwghKxQvurgatLxFcCK0vEn+SPI8qK428BZ3cpWTPr8yKizz4Zefjw4axfv76i++hqB5B/oW9mfd7AgQPZuXNnt4f07u8KLwsbOHBg2ev4qchm1ueNGjWK1tZWduzYUetUeq3Ca47L5eJiZn3egAEDyn49r+XD3WJmZpY7FxczM8udi4uZmeXOxcXMzHLnG/q9hH9Zb2a9ia9czMwsdy4uZmaWO3eLGeBuNzPLl69czMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLn4mJmZrmrWHGRtFDSdkmPFcWGSWqStCX9HZriknS9pBZJj0iaULTOjNR+i6QZRfHPSHo0rXO9JHW2DzMzq55Kvs9lEfBD4Oai2Fzgroi4VtLcND8HOB0Ymz4TgR8DEyUNA64C6oEAHpS0IiJeTG1mAWuAlUADsKqTfVg3lHrPC/hdL2bWuYpduUTEfcCuNuFGYHGaXgycVRS/OTJrgCGSRgCnAU0RsSsVlCagIS07JCLuj4ggK2Bn7WUfZmZWJdW+53JYRGwDSH8/muIjgWeL2rWmWGfx1hLxzvbRjqRZkpolNe/YsaPbB2VmZh/UU27oq0QsuhHvkoi4MSLqI6K+rq6uq6ubmVkHKnnPpZTnJY2IiG2pa2t7ircChxe1GwVsTfHJbeL3pvioEu0720eP5/sbZtZXVPvKZQVQGPE1A7i9KH5+GjU2CXg5dWmtBk6VNDSN+joVWJ2WvSppUholdn6bbZXah5mZVUnFrlwkLSW76jhUUivZqK9rgdskzQSeAc5OzVcCZwAtwBvA1wEiYpek7wLrUrvvRERhkMA3yEakDSIbJbYqxTvah5mZVUnFiktEnNvBoikl2gYwu4PtLAQWlog3A+NLxHeW2oeZmVVPT7mhb2ZmfYiLi5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeWuYq85tr5tftMTJeOXTT2mypmYWU/kKxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9zVpLhIukzSBkmPSVoqaaCkMZLWStoi6VZJB6S2B6b5lrR8dNF2Lk/xzZJOK4o3pFiLpLnVP0Izs/1b1YuLpJHAXwP1ETEe6AecA3wfmB8RY4EXgZlplZnAixFxNDA/tUPSuLTeJ4AG4EeS+knqB9wAnA6MA85Nbc3MrEpq1S3WHxgkqT/wYWAb8AVgeVq+GDgrTTemedLyKZKU4ssi4u2IeApoAU5Kn5aIeDIi3gGWpbZmZlYlVS8uEfEc8APgGbKi8jLwIPBSROxOzVqBkWl6JPBsWnd3aj+8ON5mnY7i7UiaJalZUvOOHTv2/eDMzAwos7hIGp/XDiUNJbuSGAN8DDiIrAurrSis0sGyrsbbByNujIj6iKivq6vbW+pmZlamcq9cfiLpAUkXSxqyj/v8IvBUROyIiHeBXwH/BRiSuskARgFb03QrcDhAWv4RYFdxvM06HcXNzKxKynq2WER8VtJY4C+BZkkPAD+LiKZu7PMZYJKkDwNvAlOAZuAeYBrZPZIZwO2p/Yo0f39afndEhKQVwL9Iuo7sCmgs8ADZlctYSWOA58hu+n+1G3lWTEfP5TIz6yvKfnBlRGyR9HdkheB64NPpxvoVEfGrLmxnraTlwEPAbuBh4EbgDmCZpO+l2IK0ygJgiaQWsiuWc9J2Nki6DdiYtjM7It4DkHQJsJpsJNrCiNhQbn5mZrbvyiouko4Hvg78OdAEfCkiHpL0MbIrirKLC0BEXAVc1Sb8JNlIr7Zt3wLO7mA7VwNXl4ivBFZ2JSczM8tPuVcuPwRuIrtKebMQjIit6WrGzMxsj3KLyxnAm0XdTh8CBkbEGxGxpGLZmZlZr1TuaLE7gUFF8x9OMTMzs3bKLS4DI+K1wkya/nBlUjIzs96u3OLyuqQJhRlJnyEbRmxmZtZOufdcvgX8QlLhx4gjgOmVScnMzHq7cn9EuU7SscDHyX6k+Hj6db2ZmVk7Zf+IEjgRGJ3W+bQkIuLmimRlZma9Wrk/olwC/CmwHngvhQNwcTEzs3bKvXKpB8ZFRMmnC5uZmRUrd7TYY8CfVDIRMzPrO8q9cjkU2Jiehvx2IRgRX65IVmZm1quVW1zmVTIJ6zs6ep3AZVOPqXImZlZL5Q5F/g9JRwJjI+LO9C6WfpVNzczMeqtyX3P8V8By4J9TaCTw60olZWZmvVu5N/RnA6cAr0D24jDgo5VKyszMerdyi8vbEfFOYSa9y97Dks3MrKRyi8t/SLoCGCRpKvAL4DeVS8vMzHqzcovLXGAH8Cjw38leIew3UJqZWUnljhZ7n+w1xzdVNh0zM+sLyn222FOUuMcSEUflnpGZmfV6XXm2WMFA4GxgWP7pmJlZX1DWPZeI2Fn0eS4i/hH4QoVzMzOzXqrcbrEJRbMfIruSGVyRjMzMrNcrd7TYPxR9rgE+A3yluzuVNETSckmPS9ok6WRJwyQ1SdqS/g5NbSXpekktkh4pLnSSZqT2WyTNKIp/RtKjaZ3rJam7uZqZWdeVO1rs8znv95+Af4uIaZIOAD4MXAHcFRHXSppLNvx5DnA6MDZ9JgI/BiZKGgZcRXYVFcCDklZExIupzSxgDdmw6QZgVc7HYGZmHSi3W+x/dLY8Iq4rd4eSDgE+B1yQ1n0HeEdSIzA5NVsM3EtWXBqBm9OLytakq54RqW1TROxK220CGiTdCxwSEfen+M3AWbi4mJlVTbndYvXAN8geWDkSuAgYR3bfpav3Xo4i+0HmzyQ9LOmnkg4CDouIbQDpb+HZZSOBZ4vWby3Ko6N4a4m4mZlVSVdeFjYhIl4FkDQP+EVEXNjNfU4AvhkRayX9E1kXWEdK3S+JbsTbb1iaRdZ9xhFHHNFZzmZm1gXlXrkcAbxTNP8OMLqb+2wFWiNibZpfTlZsnk/dXaS/24vaH160/ihg617io0rE24mIGyOiPiLq6+rqunk4ZmbWVrnFZQnwgKR5kq4C1gI3d2eHEfEH4FlJH0+hKcBGYAVQGPE1A7g9Ta8Azk+jxiYBL6dus9XAqZKGppFlpwKr07JXJU1Ko8TOL9qWmZlVQbmjxa6WtAr4ryn09Yh4eB/2+03gljRS7Eng62SF7jZJM4FnyJ4CANlorzOAFuCN1JaI2CXpu8C61O47hZv7ZPeHFgGDyG7k+2a+mVkVlXvPBbLhwq9ExM8k1UkaExFPdWenEbGeDz5SpmBKibZB9rKyUttZCCwsEW8Gxncntzx19D55M7O+rtzXHF9FNiz48hQaAPy8UkmZmVnvVu49l78Avgy8DhARW/HjX8zMrAPlFpd3UvdUAKTfpZiZmZVUbnG5TdI/A0Mk/RVwJ35xmJmZdaDc0WI/kDQVeAX4OHBlRDRVNDPrUzoa3HDZ1GOqnImZVcNei4ukfmS/H/ki4IJiZmZ7tddusYh4D3hD0keqkI+ZmfUB5f7O5S3g0fTk4dcLwYj464pkZWZmvVq5xeWO9DEzM9urTouLpCMi4pmIWFythMzMrPfb2z2XXxcmJP2ywrmYmVkfsbfiUvxulKMqmYiZmfUdeysu0cG0mZlZh/Z2Q/9Tkl4hu4IZlKZJ8xERh1Q0OzMz65U6LS4R0a9aifRmfrS+mdkHlftsMTMzs7K5uJiZWe5cXMzMLHcuLmZmljsXFzMzy52Li5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7mpWXCT1k/SwpN+m+TGS1kraIulWSQek+IFpviUtH120jctTfLOk04riDSnWImlutY/NzGx/V8srl0uBTUXz3wfmR8RY4EVgZorPBF6MiKOB+akdksYB5wCfABqAH6WC1Q+4ATgdGAecm9qamVmV1KS4SBoF/Dnw0zQv4AvA8tRkMXBWmm5M86TlU1L7RmBZRLwdEU8BLcBJ6dMSEU9GxDvAstTWzMyqpFZXLv8I/C3wfpofDrwUEbvTfCswMk2PBJ4FSMtfTu33xNus01G8HUmzJDVLat6xY8e+HpOZmSVVLy6SzgS2R8SDxeESTWMvy7oabx+MuDEi6iOivq6urpOszcysK/b2srBKOAX4sqQzgIHAIWRXMkMk9U9XJ6OAral9K3A40CqpP/ARYFdRvKB4nY7iZmZWBVW/comIyyNiVESMJrshf3dEnAfcA0xLzWYAt6fpFWmetPzuiIgUPyeNJhsDjAUeANYBY9PoswPSPlZU4dDMzCypxZVLR+YAyyR9D3gYWJDiC4AlklrIrljOAYiIDZJuAzYCu4HZEfEegKRLgNVAP2BhRGyo6pGYme3nalpcIuJe4N40/STZSK+2bd4Czu5g/auBq0vEVwIrc0zVzMy6wL/QNzOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLXk14WZvuh+U1PlIxfNvWYKmdiZnnylYuZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMzMLHcuLmZmljsXFzMzy13Vi4ukwyXdI2mTpA2SLk3xYZKaJG1Jf4emuCRdL6lF0iOSJhRta0Zqv0XSjKL4ZyQ9mta5XpKqfZxmZvuzWly57Ab+JiKOAyYBsyWNA+YCd0XEWOCuNA9wOjA2fWYBP4asGAFXAROBk4CrCgUptZlVtF5DFY7LzMySqheXiNgWEQ+l6VeBTcBIoBFYnJotBs5K043AzZFZAwyRNAI4DWiKiF0R8SLQBDSkZYdExP0REcDNRdsyM7MqqOk9F0mjgU8Da4HDImIbZAUI+GhqNhJ4tmi11hTrLN5aIl5q/7MkNUtq3rFjx74ejpmZJTUrLpIOBn4JfCsiXumsaYlYdCPePhhxY0TUR0R9XV3d3lI2M7My1aS4SBpAVlhuiYhfpfDzqUuL9Hd7ircChxetPgrYupf4qBJxMzOrklqMFhOwANgUEdcVLVoBFEZ8zQBuL4qfn0aNTQJeTt1mq4FTJQ1NN/JPBVanZa9KmpT2dX7RtszMrApq8T6XU4CvAY9KWp9iVwDXArdJmgk8A5ydlq0EzgBagDeArwNExC5J3wXWpXbfiYhdafobwCJgELAqfczMrEqqXlwi4j8pfV8EYEqJ9gHM7mBbC4GFJeLNwPh9SNPMzPaBf6FvZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMzMLHcuLmZmljsXFzMzy52Li5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma567PFRVKDpM2SWiTNrXU+Zmb7kz5ZXCT1A24ATgfGAedKGlfbrMzM9h/9a51AhZwEtETEkwCSlgGNwMaaZmVdMr/piXaxy6YeU4NMzKyrFBG1ziF3kqYBDRFxYZr/GjAxIi5p024WMCvNfhzY3M1dHgq80M11q6U35AjOM0+9IUfoHXn2hhyhNnkeGRF1bYN99cpFJWLtqmhE3AjcuM87k5ojon5ft1NJvSFHcJ556g05Qu/IszfkCD0rzz55zwVoBQ4vmh8FbK1RLmZm+52+WlzWAWMljZF0AHAOsKLGOZmZ7Tf6ZLdYROyWdAmwGugHLIyIDRXc5T53rVVBb8gRnGeeekOO0Dvy7A05Qg/Ks0/e0Dczs9rqq91iZmZWQy4uZmaWOxeXfdCTHzEj6WlJj0paL6k5xYZJapK0Jf0dWoO8FkraLumxoljJvJS5Pp3fRyRNqGGO8yQ9l87neklnFC27POW4WdJp1cgx7fdwSfdI2iRpg6RLU7zHnM9OcuxR51PSQEkPSPpdyvN/p/gYSWvTubw1DRBC0oFpviUtH13DHBdJeqroXJ6Q4jX5/uwREf5040M2UOD/AUcBBwC/A8bVOq+i/J4GDm0T+3tgbpqeC3y/Bnl9DpgAPLa3vIAzgFVkv1uaBKytYY7zgP9Zou249M/+QGBM+neiX5XyHAFMSNODgSdSPj3mfHaSY486n+mcHJymBwBr0zm6DTgnxX8CfCNNXwz8JE2fA9xawxwXAdNKtK/J96fw8ZVL9+15xExEvAMUHjHTkzUCi9P0YuCsaicQEfcBu9qEO8qrEbg5MmuAIZJG1CjHjjQCyyLi7Yh4Cmgh+3ej4iJiW0Q8lKZfBTYBI+lB57OTHDtSk/OZzslraXZA+gTwBWB5irc9l4VzvByYIqnUj7erkWNHavL9KXBx6b6RwLNF8610/qWptgD+XdKD6TE3AIdFxDbIvvTAR2uW3Qd1lFdPO8eXpO6FhUVdij0ix9Qt82my/5vtkeezTY7Qw86npH6S1gPbgSayq6aXImJ3iVz25JmWvwwMr3aOEVE4l1enczlf0oFtcyyRf8W5uHRfWY+YqaFTImIC2ZOhZ0v6XK0T6oaedI5/DPwpcAKwDfiHFK95jpIOBn4JfCsiXumsaYlYVXItkWOPO58R8V5EnED2RI+TgOM6yaUmebbNUdJ44HLgWOBEYBgwp5Y5Fri4dF+PfsRMRGxNf7cD/0r2ZXm+cFmc/m6vXYYf0FFePeYcR8Tz6Yv9PnATf+yqqWmOkgaQ/Uf7loj4VQr3qPNZKseeej5Tbi8B95LdpxgiqfBj8+Jc9uSZln+E8rtS88yxIXU9RkS8DfyMHnIuXVy6r8c+YkbSQZIGF6aBU4HHyPKbkZrNAG6vTYbtdJTXCuD8NOplEvByobun2tr0Vf8F2fmELMdz0uihMcBY4IEq5SRgAbApIq4rWtRjzmdHOfa08ympTtKQND0I+CLZ/aF7gGmpWdtzWTjH04C7I91Fr3KOjxf9j4TI7gkVn8vafX+qOXqgr33IRmM8QdY3++1a51OU11FkI25+B2wo5EbWJ3wXsCX9HVaD3JaSdYO8S/Z/VjM7yovssv6GdH4fBeprmOOSlMMjZF/aEUXtv51y3AycXsVz+Vmybo5HgPXpc0ZPOp+d5NijzidwPPBwyucx4MoUP4qsuLUAvwAOTPGBab4lLT+qhjnenc7lY8DP+eOIspp8fwofP/7FzMxy524xMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3/x+lcA57ZcXf/QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "part2.select('revol_util').toPandas().plot.hist(bins=50, rwidth=0.9, alpha=0.5, title='test');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_coll_amt_mean=part2.select(fn.avg('tot_coll_amt')).collect()\n",
    "part2 = part2.na.fill({'tot_coll_amt':tot_coll_amt_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_cur_bal_mean=part2.select(fn.avg('tot_cur_bal')).collect()\n",
    "part2 = part2.na.fill({'tot_cur_bal':tot_cur_bal_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_acc_6m_mean=part2.select(fn.avg('open_acc_6m')).collect()\n",
    "part2 = part2.na.fill({'open_acc_6m':open_acc_6m_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "collections_12_mths_ex_med_mean=part2.select(fn.avg('collections_12_mths_ex_med')).collect()\n",
    "part2 = part2.na.fill({'collections_12_mths_ex_med':collections_12_mths_ex_med_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "#part2.describe('tot_coll_amt').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "#part2.describe('tot_cur_bal').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#part2.describe('open_acc_6m').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#part2.groupBy(\"open_acc_6m\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # check the missing values\n",
    "# checkna_part2 = part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns])\n",
    "# checkna_part2.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do not have any missing values in these columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0        0          0           0          0                    0          0              0            0                0                0              0                   0           0                        0                0                           0                 0               0             0            0          "
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 21\n"
     ]
    }
   ],
   "source": [
    "size_shape(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.18891601882101067</td>\n",
       "      <td>0.5552736651970246</td>\n",
       "      <td>0</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_bal</td>\n",
       "      <td>1206949</td>\n",
       "      <td>16808.217865046492</td>\n",
       "      <td>22685.94006979832</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2559552.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_util</td>\n",
       "      <td>1206949</td>\n",
       "      <td>49.60215702165087</td>\n",
       "      <td>24.751729978833588</td>\n",
       "      <td>0.0</td>\n",
       "      <td>366.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_acc</td>\n",
       "      <td>1206949</td>\n",
       "      <td>23.37079445776085</td>\n",
       "      <td>11.941722677905858</td>\n",
       "      <td>2</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>initial_list_status</td>\n",
       "      <td>1206949</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>f</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp</td>\n",
       "      <td>1206949</td>\n",
       "      <td>8246.047815595506</td>\n",
       "      <td>8619.423099998028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp_inv</td>\n",
       "      <td>1206949</td>\n",
       "      <td>8244.196088209352</td>\n",
       "      <td>8618.689220779683</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>7857.2732358664325</td>\n",
       "      <td>7372.212777604972</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61947.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt_inv</td>\n",
       "      <td>1206949</td>\n",
       "      <td>7852.147555765605</td>\n",
       "      <td>7369.027624445597</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61947.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_prncp</td>\n",
       "      <td>1206949</td>\n",
       "      <td>5160.822785302468</td>\n",
       "      <td>5195.404453009808</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>1206949</td>\n",
       "      <td>2440.0498857228645</td>\n",
       "      <td>2747.206620093818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27814.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_late_fee</td>\n",
       "      <td>1206949</td>\n",
       "      <td>1.9275032905808955</td>\n",
       "      <td>12.919605093795854</td>\n",
       "      <td>-9.5E-9</td>\n",
       "      <td>1427.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>recoveries</td>\n",
       "      <td>1206949</td>\n",
       "      <td>254.47306136312878</td>\n",
       "      <td>976.7144306346049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39859.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>1206949</td>\n",
       "      <td>42.23292951247622</td>\n",
       "      <td>170.9743693190562</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7174.719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>489.148443251319</td>\n",
       "      <td>699.2390074303721</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40486.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collections_12_mths_ex_med</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.020176494615762555</td>\n",
       "      <td>0.1577334168220779</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>application_type</td>\n",
       "      <td>1206949</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Joint App</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_now_delinq</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.0035063619092438867</td>\n",
       "      <td>0.06363324149822101</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>216.8530439506285</td>\n",
       "      <td>1692.0028195280042</td>\n",
       "      <td>0.0</td>\n",
       "      <td>262740.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_cur_bal</td>\n",
       "      <td>1206949</td>\n",
       "      <td>138929.0007531387</td>\n",
       "      <td>158656.0233532199</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9971659.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.7472975245847173</td>\n",
       "      <td>1.0759859741062134</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                      1                    2           3          4\n",
       "summary                     count    mean                   stddev               min         max      \n",
       "pub_rec                     1206949  0.18891601882101067    0.5552736651970246   0           86       \n",
       "revol_bal                   1206949  16808.217865046492     22685.94006979832    0.0         2559552.0\n",
       "revol_util                  1206949  49.60215702165087      24.751729978833588   0.0         366.6    \n",
       "total_acc                   1206949  23.37079445776085      11.941722677905858   2           176      \n",
       "initial_list_status         1206949  None                   None                 f           w        \n",
       "out_prncp                   1206949  8246.047815595506      8619.423099998028    0.0         40000.0  \n",
       "out_prncp_inv               1206949  8244.196088209352      8618.689220779683    0.0         40000.0  \n",
       "total_pymnt                 1206949  7857.2732358664325     7372.212777604972    0.0         61947.53 \n",
       "total_pymnt_inv             1206949  7852.147555765605      7369.027624445597    0.0         61947.53 \n",
       "total_rec_prncp             1206949  5160.822785302468      5195.404453009808    0.0         40000.0  \n",
       "total_rec_int               1206949  2440.0498857228645     2747.206620093818    0.0         27814.3  \n",
       "total_rec_late_fee          1206949  1.9275032905808955     12.919605093795854   -9.5E-9     1427.25  \n",
       "recoveries                  1206949  254.47306136312878     976.7144306346049    0.0         39859.55 \n",
       "collection_recovery_fee     1206949  42.23292951247622      170.9743693190562    0.0         7174.719 \n",
       "last_pymnt_amnt             1206949  489.148443251319       699.2390074303721    0.0         40486.13 \n",
       "collections_12_mths_ex_med  1206949  0.020176494615762555   0.1577334168220779   0           20       \n",
       "application_type            1206949  None                   None                 Individual  Joint App\n",
       "acc_now_delinq              1206949  0.0035063619092438867  0.06363324149822101  0           7        \n",
       "tot_coll_amt                1206949  216.8530439506285      1692.0028195280042   0.0         262740.0 \n",
       "tot_cur_bal                 1206949  138929.0007531387      158656.0233532199    0.0         9971659.0\n",
       "open_acc_6m                 1206949  0.7472975245847173     1.0759859741062134   0           18       "
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now lets add the index to the part2 cleaned dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 22\n"
     ]
    }
   ],
   "source": [
    "size_shape(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.560000</td>\n",
       "      <td>84.919998</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4341.0</td>\n",
       "      <td>10.300000</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.020020</td>\n",
       "      <td>2386.020020</td>\n",
       "      <td>167.020004</td>\n",
       "      <td>167.020004</td>\n",
       "      <td>113.980003</td>\n",
       "      <td>53.040001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84.919998</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.940001</td>\n",
       "      <td>777.229980</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>12315.0</td>\n",
       "      <td>24.200001</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.750000</td>\n",
       "      <td>29387.750000</td>\n",
       "      <td>1507.109985</td>\n",
       "      <td>1507.109985</td>\n",
       "      <td>612.250000</td>\n",
       "      <td>894.859985</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>777.229980</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>1208.0</td>\n",
       "      <td>321915.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.969999</td>\n",
       "      <td>180.690002</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>4599.0</td>\n",
       "      <td>19.100000</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.209961</td>\n",
       "      <td>4787.209961</td>\n",
       "      <td>353.890015</td>\n",
       "      <td>353.890015</td>\n",
       "      <td>212.789993</td>\n",
       "      <td>141.100006</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180.690002</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>110299.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.940001</td>\n",
       "      <td>146.509995</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>5468.0</td>\n",
       "      <td>78.099998</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.929932</td>\n",
       "      <td>3831.929932</td>\n",
       "      <td>286.709991</td>\n",
       "      <td>286.709991</td>\n",
       "      <td>168.070007</td>\n",
       "      <td>118.639999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>146.509995</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>686.0</td>\n",
       "      <td>305049.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.139999</td>\n",
       "      <td>731.780029</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250.0</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>829.0</td>\n",
       "      <td>3.600000</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.019531</td>\n",
       "      <td>29339.019531</td>\n",
       "      <td>1423.209961</td>\n",
       "      <td>1423.209961</td>\n",
       "      <td>660.979980</td>\n",
       "      <td>762.229980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>731.780029</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>116007.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  loan_amnt  funded_amnt  funded_amnt_inv        term   int_rate  installment grade sub_grade home_ownership  annual_inc verification_status      loan_status pymnt_plan    dti  delinq_2yrs  inq_last_6mths  open_acc  pub_rec  revol_bal  revol_util  total_acc initial_list_status     out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0      2500.0     2500.0       2500.0            36 months  13.560000  84.919998    C     C1        RENT           55000.0     Not Verified        Not Charged Off  n          18.24  0            1               9         1        4341.0     10.300000   34         w                   2386.020020   2386.020020    167.020004   167.020004       113.980003       53.040001      0.0                 0.0         0.0                      84.919998        0                           Individual       0               0.0           16901.0      2          \n",
       "1  1      30000.0    30000.0      30000.0           60 months  18.940001  777.229980   D     D2        MORTGAGE       90000.0     Source Verified     Not Charged Off  n          26.52  0            0               13        1        12315.0    24.200001   44         w                   29387.750000  29387.750000   1507.109985  1507.109985      612.250000       894.859985     0.0                 0.0         0.0                      777.229980       0                           Individual       0               1208.0        321915.0     4          \n",
       "2  2      5000.0     5000.0       5000.0            36 months  17.969999  180.690002   D     D1        MORTGAGE       59280.0     Source Verified     Not Charged Off  n          10.51  0            0               8         0        4599.0     19.100000   13         w                   4787.209961   4787.209961    353.890015   353.890015       212.789993       141.100006     0.0                 0.0         0.0                      180.690002       0                           Individual       0               0.0           110299.0     0          \n",
       "3  3      4000.0     4000.0       4000.0            36 months  18.940001  146.509995   D     D2        MORTGAGE       92000.0     Source Verified     Not Charged Off  n          16.74  0            0               10        0        5468.0     78.099998   13         w                   3831.929932   3831.929932    286.709991   286.709991       168.070007       118.639999     0.0                 0.0         0.0                      146.509995       0                           Individual       0               686.0         305049.0     1          \n",
       "4  4      30000.0    30000.0      30000.0           60 months  16.139999  731.780029   C     C4        MORTGAGE       57250.0     Not Verified        Not Charged Off  n          26.35  0            0               12        0        829.0      3.600000    26         w                   29339.019531  29339.019531   1423.209961  1423.209961      660.979980       762.229980     0.0                 0.0         0.0                      731.780029       0                           Individual       0               0.0           116007.0     3          "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3 = part1.join(part2, \"index\", \"outer\")\n",
    "df3.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's converts string to index (labels) using the StringIndexer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# categorical_cols = ['initial_list_status','application_type']\n",
    "# stringIndexer = feature.StringIndexer(inputCols=[cols for cols in categorical_cols],\n",
    "#                                       outputCols=[\"encoded_\" + cols for cols in categorical_cols])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below codes are to check if this object works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_SI=stringIndexer.fit(part2).transform(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# part2_SI.describe('encoded_initial_list_status').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_SI.describe('encoded_application_type').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's scale the numeric variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols = [cols for cols in part2.columns if cols not in categorical_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(numeric_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It turned out that StandardScaler can only take one column as input columns so we have to Assemble numeric columns to one column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # define the VectorAssembler object\n",
    "# va = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"features\")\n",
    "# # define the StandardScaler object\n",
    "# sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"features\", outputCol=\"scaled_features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardScaler = feature.StandardScaler(withMean=True, withStd=True, inputCols=[cols for cols in numeric_cols],\n",
    "#                                       outputCols=[\"scaled_\" + cols for cols in numeric_cols])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 3 stages: stringindexer, VectorAssembler and StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_pipe=Pipeline(stages=[stringIndexer, va, sScaler])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fitted_part2 = part2_pipe.fit(part2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part2 = fitted_part2.transform(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Pyspark to read the dataset and check missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60124.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>36500.0</td>\n",
       "      <td>18124.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>372872.0</td>\n",
       "      <td>99468.0</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>94072.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>136927.0</td>\n",
       "      <td>11749.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>385183.0</td>\n",
       "      <td>36151.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>44984.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>92.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157548.0</td>\n",
       "      <td>29674.0</td>\n",
       "      <td>9300.0</td>\n",
       "      <td>32332.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mths_since_recent_inq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit hardship_flag disbursement_method debt_settlement_flag\n",
       "0  2.0                    0.0                    2.0             5.0              3.0          3.0        16.0       7.0            18.0           5.0                  9.0       0.0               0.0           0.0                 3.0                 100.0           0.0               1.0                   0.0        60124.0          16901.0            36500.0         18124.0                     N             Cash                N                  \n",
       "1  4.0                    0.0                    2.0             4.0              4.0          9.0        27.0       8.0            14.0           4.0                  13.0      0.0               0.0           0.0                 6.0                 95.0            0.0               1.0                   0.0        372872.0         99468.0            15000.0         94072.0                     N             Cash                N                  \n",
       "2  14.0                   0.0                    0.0             3.0              3.0          3.0        4.0        6.0            7.0            3.0                  8.0       0.0               0.0           0.0                 0.0                 100.0           0.0               0.0                   0.0        136927.0         11749.0            13800.0         10000.0                     N             Cash                N                  \n",
       "3  5.0                    0.0                    1.0             2.0              1.0          2.0        7.0        2.0            3.0            2.0                  10.0      0.0               0.0           0.0                 3.0                 100.0           100.0             0.0                   0.0        385183.0         36151.0            5000.0          44984.0                     N             Cash                N                  \n",
       "4  13.0                   0.0                    2.0             2.0              3.0          8.0        9.0        6.0            15.0           2.0                  12.0      0.0               0.0           0.0                 5.0                 92.3            0.0               0.0                   0.0        157548.0         29674.0            9300.0          32332.0                     N             Cash                N                  "
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part4 = current_df1.select(current_df1.columns[76::1])\n",
    "# Displaying the dataframe in pandas\n",
    "# pd.set_option('display.max_columns', None)  \n",
    "# pd.set_option('display.max_colwidth',-1)\n",
    "part4.toPandas().head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- mths_since_recent_inq: integer (nullable = true)\n",
      " |-- num_accts_ever_120_pd: integer (nullable = true)\n",
      " |-- num_actv_bc_tl: integer (nullable = true)\n",
      " |-- num_actv_rev_tl: integer (nullable = true)\n",
      " |-- num_bc_sats: integer (nullable = true)\n",
      " |-- num_bc_tl: integer (nullable = true)\n",
      " |-- num_il_tl: integer (nullable = true)\n",
      " |-- num_op_rev_tl: integer (nullable = true)\n",
      " |-- num_rev_accts: integer (nullable = true)\n",
      " |-- num_rev_tl_bal_gt_0: integer (nullable = true)\n",
      " |-- num_sats: integer (nullable = true)\n",
      " |-- num_tl_120dpd_2m: integer (nullable = true)\n",
      " |-- num_tl_30dpd: integer (nullable = true)\n",
      " |-- num_tl_90g_dpd_24m: integer (nullable = true)\n",
      " |-- num_tl_op_past_12m: integer (nullable = true)\n",
      " |-- pct_tl_nvr_dlq: double (nullable = true)\n",
      " |-- percent_bc_gt_75: double (nullable = true)\n",
      " |-- pub_rec_bankruptcies: integer (nullable = true)\n",
      " |-- tax_liens: integer (nullable = true)\n",
      " |-- tot_hi_cred_lim: integer (nullable = true)\n",
      " |-- total_bal_ex_mort: integer (nullable = true)\n",
      " |-- total_bc_limit: integer (nullable = true)\n",
      " |-- total_il_high_credit_limit: integer (nullable = true)\n",
      " |-- hardship_flag: string (nullable = true)\n",
      " |-- disbursement_method: string (nullable = true)\n",
      " |-- debt_settlement_flag: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part4.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using count distinct to check about the columns and ascertain its type accordingly\n",
    "# part4.select([fn.countDistinct(c).alias(c) for c in part4.columns]).toPandas().T\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the summary of each col\n",
    "# part4.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using count distinct to check about the columns and ascertain its type accordingly \n",
    "# - whether categorical,float ,integer..etc\n",
    "Distinctcount4 = part4.select([fn.countDistinct(c).alias(c) for c in part4.columns])\n",
    "\n",
    "# Distinct count Converting to Pandas(So it computes faster)\n",
    "DC4= Distinctcount4.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is to check for categorical numerical and see if we have weird categories.()\n",
    "# for col in part4.columns:\n",
    "#     if DC4[col][0]<1000:\n",
    "#         print(part4.groupby(col).count().show())\n",
    "#     else:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mths_since_recent_inq int\n",
      "mths_since_recent_inq  done\n",
      "mths_since_recent_inq winsorising done\n",
      "num_accts_ever_120_pd int\n",
      "num_accts_ever_120_pd  done\n",
      "num_accts_ever_120_pd winsorising done\n",
      "num_actv_bc_tl int\n",
      "num_actv_bc_tl  done\n",
      "num_actv_bc_tl winsorising done\n",
      "num_actv_rev_tl int\n",
      "num_actv_rev_tl  done\n",
      "num_actv_rev_tl winsorising done\n",
      "num_bc_sats int\n",
      "num_bc_sats  done\n",
      "num_bc_sats winsorising done\n",
      "num_bc_tl int\n",
      "num_bc_tl  done\n",
      "num_bc_tl winsorising done\n",
      "num_il_tl int\n",
      "num_il_tl  done\n",
      "num_il_tl winsorising done\n",
      "num_op_rev_tl int\n",
      "num_op_rev_tl  done\n",
      "num_op_rev_tl winsorising done\n",
      "num_rev_accts int\n",
      "num_rev_accts  done\n",
      "num_rev_accts winsorising done\n",
      "num_rev_tl_bal_gt_0 int\n",
      "num_rev_tl_bal_gt_0  done\n",
      "num_rev_tl_bal_gt_0 winsorising done\n",
      "num_sats int\n",
      "num_sats  done\n",
      "num_sats winsorising done\n",
      "num_tl_120dpd_2m int\n",
      "num_tl_120dpd_2m  done\n",
      "num_tl_120dpd_2m winsorising done\n",
      "num_tl_30dpd int\n",
      "num_tl_30dpd  done\n",
      "num_tl_30dpd winsorising done\n",
      "num_tl_90g_dpd_24m int\n",
      "num_tl_90g_dpd_24m  done\n",
      "num_tl_90g_dpd_24m winsorising done\n",
      "num_tl_op_past_12m int\n",
      "num_tl_op_past_12m  done\n",
      "num_tl_op_past_12m winsorising done\n",
      "pct_tl_nvr_dlq double\n",
      "pct_tl_nvr_dlq  done\n",
      "pct_tl_nvr_dlq winsorising done\n",
      "percent_bc_gt_75 double\n",
      "percent_bc_gt_75  done\n",
      "percent_bc_gt_75 winsorising done\n",
      "pub_rec_bankruptcies int\n",
      "pub_rec_bankruptcies  done\n",
      "pub_rec_bankruptcies winsorising done\n",
      "tax_liens int\n",
      "tax_liens  done\n",
      "tax_liens winsorising done\n",
      "tot_hi_cred_lim int\n",
      "tot_hi_cred_lim winsorising done\n",
      "total_bal_ex_mort int\n",
      "total_bal_ex_mort winsorising done\n",
      "total_bc_limit int\n",
      "total_bc_limit winsorising done\n",
      "total_il_high_credit_limit int\n",
      "total_il_high_credit_limit winsorising done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             0\n",
       "mths_since_recent_inq       0 \n",
       "num_accts_ever_120_pd       0 \n",
       "num_actv_bc_tl              0 \n",
       "num_actv_rev_tl             0 \n",
       "num_bc_sats                 0 \n",
       "num_bc_tl                   0 \n",
       "num_il_tl                   0 \n",
       "num_op_rev_tl               0 \n",
       "num_rev_accts               0 \n",
       "num_rev_tl_bal_gt_0         0 \n",
       "num_sats                    0 \n",
       "num_tl_120dpd_2m            0 \n",
       "num_tl_30dpd                0 \n",
       "num_tl_90g_dpd_24m          0 \n",
       "num_tl_op_past_12m          0 \n",
       "pct_tl_nvr_dlq              0 \n",
       "percent_bc_gt_75            0 \n",
       "pub_rec_bankruptcies        0 \n",
       "tax_liens                   0 \n",
       "tot_hi_cred_lim             0 \n",
       "total_bal_ex_mort           0 \n",
       "total_bc_limit              0 \n",
       "total_il_high_credit_limit  0 \n",
       "hardship_flag               33\n",
       "disbursement_method         34\n",
       "debt_settlement_flag        30"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# All these values along with NAs should be replaced first - by the mode for categorical columns with less than 1000 \n",
    "# distinct values\n",
    "# These are taken from the above output\n",
    "cols = ['hardship_flag','disbursement_method','debt_settlement_flag']\n",
    "wrong_val = ['Feb-2019','Individual','Oct-2011','Oct-2016']\n",
    "cleaned_int_categorical4 = []\n",
    "\n",
    "numerical_cols4 = []\n",
    "d = {}\n",
    "\n",
    "for col,dtype in part4.dtypes:\n",
    "    \n",
    "    #Checking that the distinct values less than 1000 to be qualified as a categorical where we replace the values with mode\n",
    "    if DC4[col][0]<1000:\n",
    "        if dtype in ['double','int']:\n",
    "            print(col,dtype)\n",
    "            part4 = part4.withColumn(col,fn.when(fn.col(col).isin(wrong_val),'null').otherwise(fn.col(col)))\n",
    "            \n",
    "            #Converting them to integer, since in my case.. all are integer type category\n",
    "            part4 = part4.withColumn(col,part4[col].cast(IntegerType()))\n",
    "            \n",
    "            #Filling the NAs by the mode\n",
    "            temp = part4.groupby(fn.col(col)).count()\n",
    "            \n",
    "            #Making sure that the grouped mode value is not a null\n",
    "            mode_value = temp.orderBy(temp['count'].desc()).collect()[0][0]\n",
    "            if mode_value is None:\n",
    "                mode_value = temp.orderBy(temp['count'].desc()).collect()[1][0]\n",
    "            part4 = part4.fillna(mode_value,subset = col)\n",
    "            print(col,' done')\n",
    "            cleaned_int_categorical4.append(col)\n",
    "            \n",
    "            #Winsorizing the columns\n",
    "            d[col] = part4.approxQuantile(col,[0.01,0.99],0.25)\n",
    "            part4 = part4.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "            print(col,'winsorising done')\n",
    "        else:\n",
    "            continue\n",
    "                    \n",
    "    else:\n",
    "        #This means that they have more than 1000 distinct values and can be treated as Float continous\n",
    "        print(col,dtype)\n",
    "        part4 = part4.withColumn(col,fn.when(part4[col].isin(wrong_val),'null').otherwise(part4[col]))\n",
    "        \n",
    "        #Casting to float type\n",
    "        part4 = part4.withColumn(col,part4[col].cast(FloatType()))\n",
    "        \n",
    "        #Replacing the float type value nas with its mean\n",
    "        mean_val = part4.select(fn.avg(fn.col(col))).collect()[0][0]\n",
    "        part4 = part4.fillna(mean_val,subset = col)\n",
    "        \n",
    "        #Winsorizing the values\n",
    "        d[col] = part4.approxQuantile(col,[0.01,0.99],0.25)\n",
    "        part4 = part4.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "        print(col,'winsorising done')\n",
    "        numerical_cols4.append(col)\n",
    "   \n",
    "    \n",
    "#These are the count of Null values in the columns\n",
    "part4.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part4.columns]).toPandas().T           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0\n",
       "mths_since_recent_inq       0\n",
       "num_accts_ever_120_pd       0\n",
       "num_actv_bc_tl              0\n",
       "num_actv_rev_tl             0\n",
       "num_bc_sats                 0\n",
       "num_bc_tl                   0\n",
       "num_il_tl                   0\n",
       "num_op_rev_tl               0\n",
       "num_rev_accts               0\n",
       "num_rev_tl_bal_gt_0         0\n",
       "num_sats                    0\n",
       "num_tl_120dpd_2m            0\n",
       "num_tl_30dpd                0\n",
       "num_tl_90g_dpd_24m          0\n",
       "num_tl_op_past_12m          0\n",
       "pct_tl_nvr_dlq              0\n",
       "percent_bc_gt_75            0\n",
       "pub_rec_bankruptcies        0\n",
       "tax_liens                   0\n",
       "tot_hi_cred_lim             0\n",
       "total_bal_ex_mort           0\n",
       "total_bc_limit              0\n",
       "total_il_high_credit_limit  0\n",
       "hardship_flag               0\n",
       "disbursement_method         0\n",
       "debt_settlement_flag        0"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now for the string Cols - Since those are categorical, I will replace it by the value 'unknown'. Since the fraction of Null is negligible\n",
    "string_cols4 = ['disbursement_method','debt_settlement_flag','hardship_flag']\n",
    "part4 = part4.fillna('unknown',subset = string_cols4)\n",
    "part4.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part4.columns]).toPandas().T    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleaned_dataframe_part4 = part4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets add the index to part4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "part4 = part4.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part4_pd = part4.describe().toPandas()\n",
    "# part4_pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_cols = []\n",
    "# # Now that cleaning is done ; Let us start with the pipeline\n",
    "# # Since all the categorical columns are numbers\n",
    "\n",
    "# cat_coded4_cols = []\n",
    "# string_cols4 = []\n",
    "# for c in cleaned_int_categorical4:\n",
    "#     print(c)\n",
    "#     part4 = Pipeline(stages = [feature.OneHotEncoder(inputCol = c,outputCol = c+'catcoded4')]).fit(part4).transform(part4)\n",
    "#     cat_coded4_cols.append(c+'catcoded4')\n",
    "# for c in string_cols:\n",
    "#     part4 = Pipeline(stages = [feature.StringIndexer(inputCol = c,outputCol = c+'index4'),\n",
    "#                               feature.OneHotEncoder(inputCol = c+'index4',outputCol = c+'stringcoded4')]).fit(part4).transform(part4)\n",
    "#     string_cols4.append(c+'stringcoded4')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# total_cols4 = cat_coded4_cols + string_cols4 + numerical_cols4\n",
    "\n",
    "# #Use features4 as stages for your modelling needs\n",
    "# part4_pipe = Pipeline(stages = [feature.VectorAssembler(inputCols = total_cols4,outputCol = 'feature4')])\n",
    "# part4_pipe.fit(part4).transform(part4).first()\n",
    "# #For cleaned dataframe - use cleaned_dataframe_part4 ( which is just all columns cleaned )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>12560</td>\n",
       "      <td>69</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2137</td>\n",
       "      <td>28</td>\n",
       "      <td>42000</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1878</td>\n",
       "      <td>34360.0</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>87153</td>\n",
       "      <td>88</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>998</td>\n",
       "      <td>57</td>\n",
       "      <td>50800</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24763</td>\n",
       "      <td>13761.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>7150</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>24100</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>18383</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>77.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>30683</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3761</td>\n",
       "      <td>70</td>\n",
       "      <td>7000</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30505</td>\n",
       "      <td>1239.0</td>\n",
       "      <td>75.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>28845</td>\n",
       "      <td>89</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>516</td>\n",
       "      <td>54</td>\n",
       "      <td>23100</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9667</td>\n",
       "      <td>8471.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  open_act_il open_il_12m open_il_24m mths_since_rcnt_il total_bal_il il_util open_rv_12m open_rv_24m max_bal_bc all_util total_rev_hi_lim inq_fi total_cu_tl inq_last_12m  acc_open_past_24mths avg_cur_bal  bc_open_to_buy bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc\n",
       "0  2           1           2           2                  12560        69      2           7           2137       28       42000            1      11          2            9.0                   1878        34360.0         5.9     0.0                       0.0          140.0               212.0                 1.0                    1.0             0.0       1.0                 \n",
       "1  4           2           3           3                  87153        88      4           5           998        57       50800            2      15          2            10.0                  24763       13761.0         8.3     0.0                       0.0          163.0               378.0                 4.0                    3.0             3.0       4.0                 \n",
       "2  1           0           2           14                 7150         72      0           2           0          35       24100            1      5           0            4.0                   18383       13800.0         0       0.0                       0.0          87.0                92.0                  15.0                   14.0            2.0       77.0                \n",
       "3  5           3           5           5                  30683        68      0           0           3761       70       7000             2      4           3            5.0                   30505       1239.0          75.2    0.0                       0.0          62.0                154.0                 64.0                   5.0             3.0       64.0                \n",
       "4  5           3           5           4                  28845        89      2           4           516        54       23100            1      0           0            9.0                   9667        8471.0          8.9     0.0                       0.0          53.0                216.0                 2.0                    2.0             2.0       2.0                 "
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part3 = current_df1.select(current_df1.columns[50:76:1])\n",
    "part3.toPandas().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>open_act_il</td>\n",
       "      <td>206297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>206304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>206307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>240368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>206306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>il_util</td>\n",
       "      <td>355256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>206308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max_bal_bc</td>\n",
       "      <td>206309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>all_util</td>\n",
       "      <td>206487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rev_hi_lim</td>\n",
       "      <td>10316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_fi</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_cu_tl</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_12m</td>\n",
       "      <td>206313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_open_past_24mths</td>\n",
       "      <td>6884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>avg_cur_bal</td>\n",
       "      <td>10370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_open_to_buy</td>\n",
       "      <td>21150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_util</td>\n",
       "      <td>21696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>chargeoff_within_12_mths</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_amnt</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_il_acct</td>\n",
       "      <td>50004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_rev_tl_op</td>\n",
       "      <td>10310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_rev_tl_op</td>\n",
       "      <td>10315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_tl</td>\n",
       "      <td>10314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mort_acc</td>\n",
       "      <td>6885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_bc</td>\n",
       "      <td>20333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               0\n",
       "open_act_il               206297\n",
       "open_il_12m               206304\n",
       "open_il_24m               206307\n",
       "mths_since_rcnt_il        240368\n",
       "total_bal_il              206306\n",
       "il_util                   355256\n",
       "open_rv_12m               206310\n",
       "open_rv_24m               206308\n",
       "max_bal_bc                206309\n",
       "all_util                  206487\n",
       "total_rev_hi_lim          10316 \n",
       "inq_fi                    206310\n",
       "total_cu_tl               206310\n",
       "inq_last_12m              206313\n",
       "acc_open_past_24mths      6884  \n",
       "avg_cur_bal               10370 \n",
       "bc_open_to_buy            21150 \n",
       "bc_util                   21696 \n",
       "chargeoff_within_12_mths  39    \n",
       "delinq_amnt               28    \n",
       "mo_sin_old_il_acct        50004 \n",
       "mo_sin_old_rev_tl_op      10310 \n",
       "mo_sin_rcnt_rev_tl_op     10315 \n",
       "mo_sin_rcnt_tl            10314 \n",
       "mort_acc                  6885  \n",
       "mths_since_recent_bc      20333 "
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "part3.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part3.columns]).toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using count distinct to check about the columns and ascertain its type accordingly - whether categorical,float ,integer..etc\n",
    "Distinctcount3 = part3.select([fn.countDistinct(c).alias(c) for c in part3.columns])\n",
    "# Distinctcount Converting to Pandas(So it computes faster)\n",
    "DC3= Distinctcount3.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is to check for wrong values & correct them.\n",
    "# for col in part3.columns:\n",
    "#     if Distinctcount3.select(fn.col(col)).collect()[0][0]<1000:\n",
    "#         print(part3.groupby(col).count().show())\n",
    "#     else:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "open_act_il string\n",
      "open_act_il  done\n",
      "open_act_il winsorising done\n",
      "open_il_12m string\n",
      "open_il_12m  done\n",
      "open_il_12m winsorising done\n",
      "open_il_24m string\n",
      "open_il_24m  done\n",
      "open_il_24m winsorising done\n",
      "mths_since_rcnt_il string\n",
      "mths_since_rcnt_il  done\n",
      "mths_since_rcnt_il winsorising done\n",
      "total_bal_il string\n",
      "total_bal_il winsorising done\n",
      "il_util string\n",
      "il_util  done\n",
      "il_util winsorising done\n",
      "open_rv_12m string\n",
      "open_rv_12m  done\n",
      "open_rv_12m winsorising done\n",
      "open_rv_24m string\n",
      "open_rv_24m  done\n",
      "open_rv_24m winsorising done\n",
      "max_bal_bc string\n",
      "max_bal_bc winsorising done\n",
      "all_util string\n",
      "all_util  done\n",
      "all_util winsorising done\n",
      "total_rev_hi_lim string\n",
      "total_rev_hi_lim winsorising done\n",
      "inq_fi string\n",
      "inq_fi  done\n",
      "inq_fi winsorising done\n",
      "total_cu_tl string\n",
      "total_cu_tl  done\n",
      "total_cu_tl winsorising done\n",
      "inq_last_12m string\n",
      "inq_last_12m  done\n",
      "inq_last_12m winsorising done\n",
      "acc_open_past_24mths int\n",
      "acc_open_past_24mths  done\n",
      "acc_open_past_24mths winsorising done\n",
      "avg_cur_bal string\n",
      "avg_cur_bal winsorising done\n",
      "bc_open_to_buy int\n",
      "bc_open_to_buy winsorising done\n",
      "bc_util string\n",
      "bc_util winsorising done\n",
      "chargeoff_within_12_mths double\n",
      "chargeoff_within_12_mths  done\n",
      "chargeoff_within_12_mths winsorising done\n",
      "delinq_amnt int\n",
      "delinq_amnt winsorising done\n",
      "mo_sin_old_il_acct double\n",
      "mo_sin_old_il_acct  done\n",
      "mo_sin_old_il_acct winsorising done\n",
      "mo_sin_old_rev_tl_op int\n",
      "mo_sin_old_rev_tl_op  done\n",
      "mo_sin_old_rev_tl_op winsorising done\n",
      "mo_sin_rcnt_rev_tl_op int\n",
      "mo_sin_rcnt_rev_tl_op  done\n",
      "mo_sin_rcnt_rev_tl_op winsorising done\n",
      "mo_sin_rcnt_tl int\n",
      "mo_sin_rcnt_tl  done\n",
      "mo_sin_rcnt_tl winsorising done\n",
      "mort_acc int\n",
      "mort_acc  done\n",
      "mort_acc winsorising done\n",
      "mths_since_recent_bc int\n",
      "mths_since_recent_bc  done\n",
      "mths_since_recent_bc winsorising done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>il_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max_bal_bc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>all_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rev_hi_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_fi</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_cu_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_open_past_24mths</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>avg_cur_bal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_open_to_buy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>chargeoff_within_12_mths</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_amnt</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_il_acct</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_rev_tl_op</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_rev_tl_op</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mort_acc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_bc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          0\n",
       "open_act_il               0\n",
       "open_il_12m               0\n",
       "open_il_24m               0\n",
       "mths_since_rcnt_il        0\n",
       "total_bal_il              0\n",
       "il_util                   0\n",
       "open_rv_12m               0\n",
       "open_rv_24m               0\n",
       "max_bal_bc                0\n",
       "all_util                  0\n",
       "total_rev_hi_lim          0\n",
       "inq_fi                    0\n",
       "total_cu_tl               0\n",
       "inq_last_12m              0\n",
       "acc_open_past_24mths      0\n",
       "avg_cur_bal               0\n",
       "bc_open_to_buy            0\n",
       "bc_util                   0\n",
       "chargeoff_within_12_mths  0\n",
       "delinq_amnt               0\n",
       "mo_sin_old_il_acct        0\n",
       "mo_sin_old_rev_tl_op      0\n",
       "mo_sin_rcnt_rev_tl_op     0\n",
       "mo_sin_rcnt_tl            0\n",
       "mort_acc                  0\n",
       "mths_since_recent_bc      0"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#All these values along with NAs should be replaced first - by the mode for categorical columns with less than 1000 distinct values\n",
    "#These are taken from the above output - BUT THE PROBLEM HERE IS (ONLY FROM THE TOP 20 ROWS)\n",
    "\n",
    "wrong_val = ['Feb-2019','Individual','Oct-2011','Oct-2016']\n",
    "cleaned_int_categorical3 = []\n",
    "\n",
    "numerical_cols3 = []\n",
    "d = {}\n",
    "\n",
    "for col,dtype in part3.dtypes:\n",
    "    \n",
    "    #Checking that the distinct values less than 1000 to be qualified as a categorical where we replace the values with mode\n",
    "    if DC3[col][0]<1000:\n",
    "        if dtype in ['string','double','int']:\n",
    "            print(col,dtype)\n",
    "            #COnverting those wrong values to null\n",
    "            part3 = part3.withColumn(col,fn.when(fn.col(col).isin(wrong_val),'null').otherwise(fn.col(col)))\n",
    "            \n",
    "            #Converting them to integer, since in my case.. all are integer type category\n",
    "            part3 = part3.withColumn(col,part3[col].cast(IntegerType()))\n",
    "            \n",
    "            #Filling the NAs by the mode\n",
    "            temp = part3.groupby(fn.col(col)).count()\n",
    "            \n",
    "            #Making sure that the grouped mode value is not a null\n",
    "            mode_value = temp.orderBy(temp['count'].desc()).collect()[0][0]\n",
    "            if mode_value is None:\n",
    "                mode_value = temp.orderBy(temp['count'].desc()).collect()[1][0]\n",
    "            part3 = part3.fillna(mode_value,subset = col)\n",
    "            print(col,' done')\n",
    "            cleaned_int_categorical3.append(col)\n",
    "            \n",
    "            #Winsorizing the columns\n",
    "            d[col] = part3.approxQuantile(col,[0.01,0.99],0.25)\n",
    "            part3 = part3.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "            print(col,'winsorising done')\n",
    "    else:\n",
    "        #This means that they have more than 1000 distinct values and can be treated as continous\n",
    "        print(col,dtype)\n",
    "        part3 = part3.withColumn(col,fn.when(part3[col].isin(wrong_val),'null').otherwise(part3[col]))\n",
    "        #Casting to float type\n",
    "        part3 = part3.withColumn(col,part3[col].cast(FloatType()))\n",
    "        #Replacing the float type value nas with its mean\n",
    "        mean_val = part3.select(fn.avg(fn.col(col))).collect()[0][0]\n",
    "        part3 = part3.fillna(mean_val,subset = col)\n",
    "        #Winsorizing the values\n",
    "        d[col] = part3.approxQuantile(col,[0.01,0.99],0.25)\n",
    "        part3 = part3.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "        print(col,'winsorising done')\n",
    "        numerical_cols3.append(col)\n",
    "   \n",
    "    \n",
    "#These are the count of Null values in the columns\n",
    "part3.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part3.columns]).toPandas().T           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets create an index for part 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "part3 = part3.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets join part 3 and part 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12560.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2137.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1878.0</td>\n",
       "      <td>34360.0</td>\n",
       "      <td>5.900000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60124.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>36500.0</td>\n",
       "      <td>18124.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>87153.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>998.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>50800.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24763.0</td>\n",
       "      <td>13761.0</td>\n",
       "      <td>8.300000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>372872.0</td>\n",
       "      <td>99468.0</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>94072.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7150.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>24100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>18383.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>136927.0</td>\n",
       "      <td>11749.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30683.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3761.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30505.0</td>\n",
       "      <td>1239.0</td>\n",
       "      <td>75.199997</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>385183.0</td>\n",
       "      <td>36151.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>44984.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28845.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>516.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>23100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9667.0</td>\n",
       "      <td>8471.0</td>\n",
       "      <td>8.900000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157548.0</td>\n",
       "      <td>29674.0</td>\n",
       "      <td>9300.0</td>\n",
       "      <td>32332.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  open_act_il  open_il_12m  open_il_24m  mths_since_rcnt_il  total_bal_il  il_util  open_rv_12m  open_rv_24m  max_bal_bc  all_util  total_rev_hi_lim  inq_fi  total_cu_tl  inq_last_12m  acc_open_past_24mths  avg_cur_bal  bc_open_to_buy    bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc  mths_since_recent_inq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit hardship_flag disbursement_method debt_settlement_flag\n",
       "0  0      2.0          1.0          2.0          2.0                 12560.0       69.0     2.0          7.0          2137.0      28.0      42000.0           1.0     11.0         2.0           9.0                   1878.0       34360.0         5.900000   0.0                       0.0          140.0               212.0                 1.0                    1.0             0.0       1.0                   2.0                    0.0                    2.0             5.0              3.0          3.0        16.0       7.0            18.0           5.0                  9.0       0.0               0.0           0.0                 3.0                 100.0           0.0               1.0                   0.0        60124.0          16901.0            36500.0         18124.0                     N             Cash                N                  \n",
       "1  1      4.0          2.0          3.0          3.0                 87153.0       88.0     4.0          5.0          998.0       57.0      50800.0           2.0     15.0         2.0           10.0                  24763.0      13761.0         8.300000   0.0                       0.0          163.0               378.0                 4.0                    3.0             3.0       4.0                   4.0                    0.0                    2.0             4.0              4.0          9.0        27.0       8.0            14.0           4.0                  13.0      0.0               0.0           0.0                 6.0                 95.0            0.0               1.0                   0.0        372872.0         99468.0            15000.0         94072.0                     N             Cash                N                  \n",
       "2  2      1.0          0.0          2.0          14.0                7150.0        72.0     0.0          2.0          0.0         35.0      24100.0           1.0     5.0          0.0           4.0                   18383.0      13800.0         0.000000   0.0                       0.0          87.0                92.0                  15.0                   14.0            2.0       77.0                  14.0                   0.0                    0.0             3.0              3.0          3.0        4.0        6.0            7.0            3.0                  8.0       0.0               0.0           0.0                 0.0                 100.0           0.0               0.0                   0.0        136927.0         11749.0            13800.0         10000.0                     N             Cash                N                  \n",
       "3  3      5.0          3.0          5.0          5.0                 30683.0       68.0     0.0          0.0          3761.0      70.0      7000.0            2.0     4.0          3.0           5.0                   30505.0      1239.0          75.199997  0.0                       0.0          62.0                154.0                 64.0                   5.0             3.0       64.0                  5.0                    0.0                    1.0             2.0              1.0          2.0        7.0        2.0            3.0            2.0                  10.0      0.0               0.0           0.0                 3.0                 100.0           100.0             0.0                   0.0        385183.0         36151.0            5000.0          44984.0                     N             Cash                N                  \n",
       "4  4      5.0          3.0          5.0          4.0                 28845.0       89.0     2.0          4.0          516.0       54.0      23100.0           1.0     0.0          0.0           9.0                   9667.0       8471.0          8.900000   0.0                       0.0          53.0                216.0                 2.0                    2.0             2.0       2.0                   13.0                   0.0                    2.0             2.0              3.0          8.0        9.0        6.0            15.0           2.0                  12.0      0.0               0.0           0.0                 5.0                 92.0            0.0               0.0                   0.0        157548.0         29674.0            9300.0          32332.0                     N             Cash                N                  "
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4 = part3.join(part4, \"index\", \"outer\")\n",
    "df4.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets join all parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "allParts = df3.join(df4, \"index\", \"outer\").drop(\"index\")\n",
    "# allParts.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[loan_amnt: bigint, funded_amnt: bigint, funded_amnt_inv: bigint, term: bigint, int_rate: bigint, installment: bigint, grade: bigint, sub_grade: bigint, home_ownership: bigint, annual_inc: bigint, verification_status: bigint, loan_status: bigint, pymnt_plan: bigint, dti: bigint, delinq_2yrs: bigint, inq_last_6mths: bigint, open_acc: bigint, pub_rec: bigint, revol_bal: bigint, revol_util: bigint, total_acc: bigint, initial_list_status: bigint, out_prncp: bigint, out_prncp_inv: bigint, total_pymnt: bigint, total_pymnt_inv: bigint, total_rec_prncp: bigint, total_rec_int: bigint, total_rec_late_fee: bigint, recoveries: bigint, collection_recovery_fee: bigint, last_pymnt_amnt: bigint, collections_12_mths_ex_med: bigint, application_type: bigint, acc_now_delinq: bigint, tot_coll_amt: bigint, tot_cur_bal: bigint, open_acc_6m: bigint, open_act_il: bigint, open_il_12m: bigint, open_il_24m: bigint, mths_since_rcnt_il: bigint, total_bal_il: bigint, il_util: bigint, open_rv_12m: bigint, open_rv_24m: bigint, max_bal_bc: bigint, all_util: bigint, total_rev_hi_lim: bigint, inq_fi: bigint, total_cu_tl: bigint, inq_last_12m: bigint, acc_open_past_24mths: bigint, avg_cur_bal: bigint, bc_open_to_buy: bigint, bc_util: bigint, chargeoff_within_12_mths: bigint, delinq_amnt: bigint, mo_sin_old_il_acct: bigint, mo_sin_old_rev_tl_op: bigint, mo_sin_rcnt_rev_tl_op: bigint, mo_sin_rcnt_tl: bigint, mort_acc: bigint, mths_since_recent_bc: bigint, mths_since_recent_inq: bigint, num_accts_ever_120_pd: bigint, num_actv_bc_tl: bigint, num_actv_rev_tl: bigint, num_bc_sats: bigint, num_bc_tl: bigint, num_il_tl: bigint, num_op_rev_tl: bigint, num_rev_accts: bigint, num_rev_tl_bal_gt_0: bigint, num_sats: bigint, num_tl_120dpd_2m: bigint, num_tl_30dpd: bigint, num_tl_90g_dpd_24m: bigint, num_tl_op_past_12m: bigint, pct_tl_nvr_dlq: bigint, percent_bc_gt_75: bigint, pub_rec_bankruptcies: bigint, tax_liens: bigint, tot_hi_cred_lim: bigint, total_bal_ex_mort: bigint, total_bc_limit: bigint, total_il_high_credit_limit: bigint, hardship_flag: bigint, disbursement_method: bigint, debt_settlement_flag: bigint]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns])\n",
    "# allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 90\n"
     ]
    }
   ],
   "source": [
    "# Check the dimensions of the dataframe\n",
    "size_shape(allParts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop to rows where the Loan_Status is == \"Late\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop rows that have null values\n",
    "\n",
    "allParts = allParts.filter(allParts[\"recoveries\"]. isNotNull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[loan_amnt: bigint, funded_amnt: bigint, funded_amnt_inv: bigint, term: bigint, int_rate: bigint, installment: bigint, grade: bigint, sub_grade: bigint, home_ownership: bigint, annual_inc: bigint, verification_status: bigint, loan_status: bigint, pymnt_plan: bigint, dti: bigint, delinq_2yrs: bigint, inq_last_6mths: bigint, open_acc: bigint, pub_rec: bigint, revol_bal: bigint, revol_util: bigint, total_acc: bigint, initial_list_status: bigint, out_prncp: bigint, out_prncp_inv: bigint, total_pymnt: bigint, total_pymnt_inv: bigint, total_rec_prncp: bigint, total_rec_int: bigint, total_rec_late_fee: bigint, recoveries: bigint, collection_recovery_fee: bigint, last_pymnt_amnt: bigint, collections_12_mths_ex_med: bigint, application_type: bigint, acc_now_delinq: bigint, tot_coll_amt: bigint, tot_cur_bal: bigint, open_acc_6m: bigint, open_act_il: bigint, open_il_12m: bigint, open_il_24m: bigint, mths_since_rcnt_il: bigint, total_bal_il: bigint, il_util: bigint, open_rv_12m: bigint, open_rv_24m: bigint, max_bal_bc: bigint, all_util: bigint, total_rev_hi_lim: bigint, inq_fi: bigint, total_cu_tl: bigint, inq_last_12m: bigint, acc_open_past_24mths: bigint, avg_cur_bal: bigint, bc_open_to_buy: bigint, bc_util: bigint, chargeoff_within_12_mths: bigint, delinq_amnt: bigint, mo_sin_old_il_acct: bigint, mo_sin_old_rev_tl_op: bigint, mo_sin_rcnt_rev_tl_op: bigint, mo_sin_rcnt_tl: bigint, mort_acc: bigint, mths_since_recent_bc: bigint, mths_since_recent_inq: bigint, num_accts_ever_120_pd: bigint, num_actv_bc_tl: bigint, num_actv_rev_tl: bigint, num_bc_sats: bigint, num_bc_tl: bigint, num_il_tl: bigint, num_op_rev_tl: bigint, num_rev_accts: bigint, num_rev_tl_bal_gt_0: bigint, num_sats: bigint, num_tl_120dpd_2m: bigint, num_tl_30dpd: bigint, num_tl_90g_dpd_24m: bigint, num_tl_op_past_12m: bigint, pct_tl_nvr_dlq: bigint, percent_bc_gt_75: bigint, pub_rec_bankruptcies: bigint, tax_liens: bigint, tot_hi_cred_lim: bigint, total_bal_ex_mort: bigint, total_bc_limit: bigint, total_il_high_credit_limit: bigint, hardship_flag: bigint, disbursement_method: bigint, debt_settlement_flag: bigint]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns])\n",
    "\n",
    "#allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 90\n"
     ]
    }
   ],
   "source": [
    "# Check the dimensions of the dataframe\n",
    "size_shape(allParts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts.describe(string_cols4).show()\n",
    "# print(allParts.groupby(string_cols4[0]).count().show())\n",
    "# print(allParts.groupby(string_cols4[1]).count().show())\n",
    "# print(allParts.groupby(string_cols4[2]).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts.describe(cleaned_int_categorical4).toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numerical_cols4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From doing the correlation matrix, we notice that there are some columns that need to be removed from our "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 90\n",
      "None\n",
      "rows: 1206949\n",
      "columns: 72\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "del_cols = ['policy_code','pymnt_plan','url','application_type','acc_now_delinq','emp_title','zip_code','title',\n",
    "        'installment',\n",
    "'total_rec_prncp',\n",
    "'total_rec_int',\n",
    "'recoveries',\n",
    "'inq_fi',\n",
    "'inq_last_12m',\n",
    "'chargeoff_within_12_mths',\n",
    "'num_actv_rev_tl',\n",
    "'num_bc_tl',\n",
    "'num_op_rev_tl',\n",
    "'num_tl_op_past_12m',\n",
    "'percent_bc_gt_75',\n",
    "'avg_cur_bal',\n",
    "'total_bc_limit',\n",
    "'total_il_high_credit_limit']\n",
    "allParts_dropped = allParts.drop(*del_cols)\n",
    "print(size_shape(allParts))\n",
    "print(size_shape(allParts_dropped))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Here we will make a way to combine all of our work into a logical sequence so that categories can be changed using string indexer and numerical columns can be changed by using standard scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Categorical operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The column that we are going to be predicting on needs to change to a numerical value\n",
    "en_pred = feature.StringIndexer(handleInvalid='skip', inputCol='loan_status', outputCol='encoded_loan_status')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert string to index label using the StringIndexer\n",
    "cat_col_list=['term','grade','verification_status','pymnt_plan']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['initial_list_status', 'term', 'grade', 'verification_status']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['initial_list_status', 'term', 'grade', 'verification_status']"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_cols = ['initial_list_status','application_type']\n",
    "for i in cat_col_list:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols.append(i)\n",
    "categorical_cols_new = []\n",
    "for i in categorical_cols:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols_new.append(i)\n",
    "categorical_cols = categorical_cols_new\n",
    "\n",
    "print(categorical_cols)\n",
    "\n",
    "stringIndexer = feature.StringIndexer(handleInvalid='skip', inputCols=[cols for cols in categorical_cols],\n",
    "                                      outputCols=[\"encoded_\" + cols for cols in categorical_cols])\n",
    "\n",
    "categorical_cols_new = []\n",
    "for i in categorical_cols:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols_new.append(i)\n",
    "        \n",
    "        \n",
    "categorical_cols = categorical_cols_new\n",
    "#categorical_cols = categorical_cols[0:4]\n",
    "categorical_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use this feature on columns that are categorical and not ordinal\n",
    "one_hot_categorical_cols = []\n",
    "ones = ['sub_grade','home_ownership']\n",
    "\n",
    "for i in string_cols4:\n",
    "    one_hot_categorical_cols.append(i)\n",
    "\n",
    "\n",
    "string_hot = feature.StringIndexer(handleInvalid='skip', inputCols= [cols for cols in ones],\n",
    "                                      outputCols=[\"encoded_\" + cols for cols in ones])\n",
    "    \n",
    "onehot= feature.OneHotEncoder(inputCols=[\"encoded_\" + cols for cols in ones],\n",
    "                                      outputCols=[\"onehot_encoded_\" + cols for cols in ones])\n",
    "\n",
    "catHotVA = feature.VectorAssembler(inputCols=[\"onehot_encoded_\" + cols for cols in ones], outputCol=\"catHotfeatures\") # \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loan_status',\n",
       " 'initial_list_status',\n",
       " 'term',\n",
       " 'grade',\n",
       " 'verification_status',\n",
       " 'disbursement_method',\n",
       " 'debt_settlement_flag',\n",
       " 'hardship_flag']"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colsNonNumeric = [\"loan_status\"]\n",
    "    \n",
    "for i in categorical_cols:\n",
    "    if i == 'loan_status':\n",
    "        continue\n",
    "    else:\n",
    "        colsNonNumeric.append(i)\n",
    "    \n",
    "for i in one_hot_categorical_cols:\n",
    "    if i == 'loan_status':\n",
    "        continue\n",
    "    else:\n",
    "        colsNonNumeric.append(i)\n",
    "    \n",
    "\n",
    "colsNonNumeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# colsNonNumeric = [\"loan_status\"]\n",
    "    \n",
    "# for i in categorical_cols:\n",
    "#     colsNonNumeric.append(i)\n",
    "    \n",
    "# for i in one_hot_categorical_cols:\n",
    "#     colsNonNumeric.append(i)\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Need to get all the names of the encoded columns and put them into a list except for the enconded_loan_status\n",
    "# en_colsNonNumeric = []  # 'onehot_encoded_sub_grade','onehot_encoded_home_ownership'\n",
    "    \n",
    "# for i in categorical_cols:\n",
    "#     en_colsNonNumeric.append(\"encoded_\"+i)\n",
    "    \n",
    "# for i in one_hot_categorical_cols:\n",
    "#     en_colsNonNumeric.append(\"encoded_\"+i)\n",
    "    \n",
    "\n",
    "\n",
    "# # catCols  = [\"encoded_\" + cols for cols in one_hot_categorical_cols]\n",
    "\n",
    "# # oneCols = [\"encoded_\" + cols for cols in categorical_cols]\n",
    "\n",
    "# print(len(en_colsNonNumeric))\n",
    "# en_colsNonNumeric\n",
    "# del en_colsNonNumeric[6:8:1]\n",
    "# print(len(en_colsNonNumeric))\n",
    "# en_colsNonNumeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Numerical Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's scale the numeric variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric_cols = [cols for cols in allParts.columns if (cols not in colsNonNumeric) and (cols not in del_cols) and (cols not in ones)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols = [cols for cols in allParts.columns if cols not in colsNonNumeric]\n",
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It turned out that StandardScaler can only take one column as input columns so we have to Assemble numeric columns to one column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62\n",
      "rows: 1206949\n",
      "columns: 90\n"
     ]
    }
   ],
   "source": [
    "print(len(numeric_cols))\n",
    "size_shape(allParts)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the VectorAssembler object\n",
    "numericVA = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"numfeatures\")\n",
    "\n",
    "#\n",
    "catVA = feature.VectorAssembler(inputCols=[\"encoded_\" + cols for cols in categorical_cols], outputCol=\"catfeatures\") # \n",
    "\n",
    "# define the StandardScaler object\n",
    "sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"numfeatures\", outputCol=\"scaled_features\")\n",
    "\n",
    "# define a Vector assembler that handles all features that need to be scaled\n",
    "allVA = feature.VectorAssembler(inputCols=['scaled_features','catHotfeatures','catfeatures'], outputCol='final_features') # 'catfeatures'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 9 stages: stringindexer, VectorAssembler(numeric), StandardScaler, stringIndexer(one_hot_categorical),OneHotEnconder(one_hot_categroical),VectorAssembler(one_hot_categorical),VectorAssembler(categorical), stringIndexer(prediction_col), VectorAssembler(all columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "allParts_pipe=Pipeline(stages=[stringIndexer, numericVA, sScaler, string_hot, onehot,catHotVA, catVA, en_pred, allVA])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts_pipe=Pipeline(stages=[stringIndexer, onehot, numericVA, sScaler, catVA, en_pred, allVA])\n",
    "# catVA = feature.VectorAssembler(inputCols=[cols for cols in en_colsNonNumeric], outputCol=\"catfeatures\")\n",
    "# allParts_pipe=Pipeline(stages=[stringIndexer, numericVA, sScaler, en_pred, allVA])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All variables in the model \n",
    "fitted_allParts = allParts_pipe.fit(allParts)\n",
    "\n",
    "# Reduced variables in the model\n",
    "fitted_allParts_dropped = allParts_pipe.fit(allParts_dropped)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# transformed_allParts = fitted_allParts.transform(allParts)\n",
    "# transformed_allParts.limit(5).toPandas()\n",
    "\n",
    "transformed_allParts_dropped = fitted_allParts_dropped.transform(allParts_dropped)\n",
    "# transformed_allParts_dropped.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest For All Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>tot_cur_bal</td>\n",
       "      <td>0.164938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>revol_bal</td>\n",
       "      <td>0.121489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>pub_rec</td>\n",
       "      <td>0.114398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>0.095208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>0.069559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>0.057310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>collections_12_mths_ex_med</td>\n",
       "      <td>0.054438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>out_prncp</td>\n",
       "      <td>0.038960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>0.038762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0.029312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>0.028881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>0.021118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>grade</td>\n",
       "      <td>0.019895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>total_acc</td>\n",
       "      <td>0.013317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>0.012403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>0.009550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>0.009211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>0.008743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>revol_util</td>\n",
       "      <td>0.007586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>verification_status</td>\n",
       "      <td>0.006786</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        column    weight\n",
       "27  tot_cur_bal                 0.164938\n",
       "14  revol_bal                   0.121489\n",
       "13  pub_rec                     0.114398\n",
       "32  mths_since_rcnt_il          0.095208\n",
       "28  open_acc_6m                 0.069559\n",
       "33  total_bal_il                0.057310\n",
       "25  collections_12_mths_ex_med  0.054438\n",
       "18  out_prncp                   0.038960\n",
       "24  last_pymnt_amnt             0.038762\n",
       "29  open_act_il                 0.029312\n",
       "23  collection_recovery_fee     0.028881\n",
       "35  open_rv_12m                 0.021118\n",
       "3   grade                       0.019895\n",
       "16  total_acc                   0.013317\n",
       "30  open_il_12m                 0.012403\n",
       "26  tot_coll_amt                0.009550\n",
       "31  open_il_24m                 0.009211\n",
       "61  num_tl_90g_dpd_24m          0.008743\n",
       "15  revol_util                  0.007586\n",
       "7   verification_status         0.006786"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainR , testR = allParts_dropped.randomSplit([0.8,0.2],0)\n",
    "\n",
    "rfR = classification.RandomForestClassifier(featuresCol = 'final_features',labelCol = 'encoded_loan_status',maxBins = 40,maxDepth = 7)\n",
    "rf_modelR = Pipeline(stages = [allParts_pipe,rfR])\n",
    "fitted_rf_pipelineR = rf_modelR.fit(trainR)\n",
    "rf_modelR = fitted_rf_pipelineR.stages[-1]\n",
    "\n",
    "bceR = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bceR.evaluate(fitted_rf_pipelineR.transform(testR))\n",
    "\n",
    "\n",
    "pd.DataFrame(list(zip(allParts_dropped.columns[2:], rf_modelR.featureImportances.toArray())),\n",
    "            columns = ['column', 'weight']).sort_values('weight',ascending = False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9005228584865733"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bceR = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bceR.evaluate(fitted_rf_pipelineR.transform(testR))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting only the 20 features which shall be used for further evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 21\n"
     ]
    }
   ],
   "source": [
    "col20_df = pd.DataFrame(list(zip(allParts_dropped.columns[2:], rf_modelR.featureImportances.toArray())),\n",
    "            columns = ['column', 'weight']).sort_values('weight',ascending = False).head(20)\n",
    "\n",
    "\n",
    "# Creating a DataFrame with only these 20columns to do further analysis\n",
    "new_cols = list(col20_df['column'])\n",
    "spark20_df = allParts_dropped.select([col for col in new_cols]+['loan_status'])\n",
    "size_shape(spark20_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tot_cur_bal',\n",
       " 'revol_bal',\n",
       " 'pub_rec',\n",
       " 'mths_since_rcnt_il',\n",
       " 'open_acc_6m',\n",
       " 'total_bal_il',\n",
       " 'collections_12_mths_ex_med',\n",
       " 'out_prncp',\n",
       " 'last_pymnt_amnt',\n",
       " 'open_act_il',\n",
       " 'collection_recovery_fee',\n",
       " 'open_rv_12m',\n",
       " 'grade',\n",
       " 'total_acc',\n",
       " 'open_il_12m',\n",
       " 'tot_coll_amt',\n",
       " 'open_il_24m',\n",
       " 'num_tl_90g_dpd_24m',\n",
       " 'revol_util',\n",
       " 'verification_status',\n",
       " 'loan_status']"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark20_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "# spark20_df.limit(5).toPandas()\n",
    "spark20_df = spark20_df.drop(\"verification_status\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest  with TOP 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "revised_train , revised_test = spark20_df.randomSplit([0.8,0.2],0)\n",
    "revised_pipe = Pipeline(stages = [feature.StringIndexer(inputCol = 'grade',outputCol = 'grade_coded'),\n",
    "                                  feature.StringIndexer(handleInvalid='skip', inputCol='loan_status', outputCol='encoded_loan_status'),\n",
    "                                  feature.VectorAssembler(inputCols=[cols for cols in new_cols if cols != 'grade']+['grade_coded'],outputCol = 'features') ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "ename": "IllegalArgumentException",
     "evalue": "verification_status does not exist. Available: tot_cur_bal, revol_bal, pub_rec, mths_since_rcnt_il, open_acc_6m, total_bal_il, collections_12_mths_ex_med, out_prncp, last_pymnt_amnt, open_act_il, collection_recovery_fee, open_rv_12m, grade, total_acc, open_il_12m, tot_coll_amt, open_il_24m, num_tl_90g_dpd_24m, revol_util, loan_status, grade_coded, encoded_loan_status",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\sql\\utils.py\u001b[0m in \u001b[0;36mdeco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m     97\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 98\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     99\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\lib\\py4j-0.10.8.1-src.zip\\py4j\\protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m    327\u001b[0m                     \u001b[1;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 328\u001b[1;33m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[0;32m    329\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling o13912.transform.\n: java.lang.IllegalArgumentException: verification_status does not exist. Available: tot_cur_bal, revol_bal, pub_rec, mths_since_rcnt_il, open_acc_6m, total_bal_il, collections_12_mths_ex_med, out_prncp, last_pymnt_amnt, open_act_il, collection_recovery_fee, open_rv_12m, grade, total_acc, open_il_12m, tot_coll_amt, open_il_24m, num_tl_90g_dpd_24m, revol_util, loan_status, grade_coded, encoded_loan_status\r\n\tat org.apache.spark.sql.types.StructType.$anonfun$apply$1(StructType.scala:275)\r\n\tat scala.collection.MapLike.getOrElse(MapLike.scala:131)\r\n\tat scala.collection.MapLike.getOrElse$(MapLike.scala:129)\r\n\tat scala.collection.AbstractMap.getOrElse(Map.scala:63)\r\n\tat org.apache.spark.sql.types.StructType.apply(StructType.scala:274)\r\n\tat org.apache.spark.ml.feature.VectorAssembler.$anonfun$transformSchema$1(VectorAssembler.scala:161)\r\n\tat scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245)\r\n\tat scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\r\n\tat scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\r\n\tat scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\r\n\tat scala.collection.TraversableLike.flatMap(TraversableLike.scala:245)\r\n\tat scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242)\r\n\tat scala.collection.mutable.ArrayOps$ofRef.flatMap(ArrayOps.scala:198)\r\n\tat org.apache.spark.ml.feature.VectorAssembler.transformSchema(VectorAssembler.scala:160)\r\n\tat org.apache.spark.ml.PipelineStage.transformSchema(Pipeline.scala:75)\r\n\tat org.apache.spark.ml.feature.VectorAssembler.transform(VectorAssembler.scala:85)\r\n\tat sun.reflect.GeneratedMethodAccessor226.invoke(Unknown Source)\r\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n\tat java.lang.reflect.Method.invoke(Method.java:498)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mIllegalArgumentException\u001b[0m                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-223-1b0b923ed5c9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mrevised_rF\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassification\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRandomForestClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeaturesCol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'features'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlabelCol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'encoded_loan_status'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mrf_modelR2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPipeline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstages\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mrevised_pipe\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mrevised_rF\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mfitted_rf_pipelineR2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrf_modelR2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrevised_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[1;31m# rf_modelR = fitted_rf_pipelineR2.stages[-1]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mbceR\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBinaryClassificationEvaluator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabelCol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'encoded_loan_status'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\base.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, dataset, params)\u001b[0m\n\u001b[0;32m    129\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m             raise ValueError(\"Params must be either a param map or a list/tuple of param maps, \"\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    110\u001b[0m                     \u001b[0mtransformers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mindexOfLastEstimator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 112\u001b[1;33m                         \u001b[0mdataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    113\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m                 \u001b[0mtransformers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\base.py\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, dataset, params)\u001b[0m\n\u001b[0;32m    170\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    173\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Params must be a param map but got %s.\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\pipeline.py\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    260\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    261\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstages\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 262\u001b[1;33m             \u001b[0mdataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    263\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mdataset\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\base.py\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, dataset, params)\u001b[0m\n\u001b[0;32m    170\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    173\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Params must be a param map but got %s.\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\ml\\wrapper.py\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    336\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transfer_params_to_java\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 338\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_java_obj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msql_ctx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    339\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    340\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\lib\\py4j-0.10.8.1-src.zip\\py4j\\java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m   1284\u001b[0m         \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1285\u001b[0m         return_value = get_return_value(\n\u001b[1;32m-> 1286\u001b[1;33m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[0;32m   1287\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1288\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\spark\\spark\\python\\pyspark\\sql\\utils.py\u001b[0m in \u001b[0;36mdeco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m    100\u001b[0m             \u001b[0mconverted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconvert_exception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconverted\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mUnknownException\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mconverted\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m                 \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIllegalArgumentException\u001b[0m: verification_status does not exist. Available: tot_cur_bal, revol_bal, pub_rec, mths_since_rcnt_il, open_acc_6m, total_bal_il, collections_12_mths_ex_med, out_prncp, last_pymnt_amnt, open_act_il, collection_recovery_fee, open_rv_12m, grade, total_acc, open_il_12m, tot_coll_amt, open_il_24m, num_tl_90g_dpd_24m, revol_util, loan_status, grade_coded, encoded_loan_status"
     ]
    }
   ],
   "source": [
    "#Since most here are numerical & float\n",
    "\n",
    "\n",
    "revised_rF = classification.RandomForestClassifier(featuresCol = 'features',labelCol = 'encoded_loan_status')\n",
    "rf_modelR2 = Pipeline(stages = [revised_pipe,revised_rF])\n",
    "fitted_rf_pipelineR2 = rf_modelR2.fit(revised_train)\n",
    "# rf_modelR = fitted_rf_pipelineR2.stages[-1]\n",
    "bceR = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bceR.evaluate(fitted_rf_pipelineR2.transform(revised_test))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression with TOP 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "revised_Lr = classification.LogisticRegression(featuresCol = 'features',labelCol = 'encoded_loan_status')\n",
    "paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(revised_Lr.elasticNetParam, [0., 0.01]) \\\n",
    "    .addGrid(revised_Lr.regParam, [0.1, 0.001]) \\\n",
    "    .build()\n",
    "\n",
    "#Will take 30min to train.. so care\n",
    "Lr_modelR2 = Pipeline(stages = [revised_pipe,revised_Lr])\n",
    "evaluator = BinaryClassificationEvaluator(labelCol=revised_Lr.getLabelCol(), rawPredictionCol=revised_Lr.getRawPredictionCol(), numBins=40)\n",
    "crossval = CrossValidator(estimator=Lr_modelR2, \n",
    "                          estimatorParamMaps=paramGrid, \n",
    "                          evaluator=evaluator, \n",
    "                          numFolds=2)\n",
    "final_model_fitted_Lr = crossval.fit(revised_train)\n",
    "\n",
    "bceR = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bceR.evaluate(final_model_fitted_Lr.transform(revised_test))\n",
    "\n",
    "# Full Model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Gradiant Boosting Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbtR = classification.GBTClassifier(featuresCol = 'features',labelCol = 'encoded_loan_status',maxBins = 40)\n",
    "gbt_modelR = Pipeline(stages = [revised_pipe,gbtR])\n",
    "fitted_gbt_pipelineR = gbt_modelR.fit(revised_train)\n",
    "\n",
    "\n",
    "\n",
    "bceR = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bceR.evaluate(fitted_gbt_pipelineR.transform(revised_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let us check all the models and we shall compare and contrast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The labels be\n",
    "Labels = fitted_rf_pipelineR2.stages[0].stages[1].labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "models = [fitted_rf_pipelineR2,final_model_fitted_Lr,fitted_gbt_pipelineR]\n",
    "names = ['Random Forest Model with TOP 20','Logistic Regression with TOP 20','GBT with top 20']\n",
    "for i,model in enumerate(models):\n",
    "    print(names[i])\n",
    "    tranformed_test = model.transform(revised_test)\n",
    "    y_true = tranformed_test.select(['encoded_loan_status']).collect()\n",
    "    y_pred = tranformed_test.select(['prediction']).collect()\n",
    "    print(classification_report(y_true, y_pred))\n",
    "##Here 0 is Not Charged Off, 1 is Charged Off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,model in enumerate(models):\n",
    "    print(names[i])\n",
    "    print('AUC score is '+str(bceR.evaluate(model.transform(revised_test))))\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}