std-scalar-all.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale Features and Build Model\n",
    "### Scales Raw Features\n",
    "\n",
    "Import CSV of Aggregated Darshan Logs <br>\n",
    "Apply Log10 and Percent Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./raws.csv\",lineterminator='\\n',sep = ',' ,error_bad_lines=False)\n",
    "#df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df.columns[0],axis = 1)\n",
    "df = df.drop(df.columns[0],axis = 1)\n",
    "f = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df = df.dropna(axis=0, how='any')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#files\n",
    "f['p_files'] = df['posix_number_of_files']  + 1\n",
    "f['l_files'] = df['lustre_number_of_files']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#accesses\n",
    "df['p_accesses'] = df['posix_reads'] + df['posix_writes']\n",
    "f['p_accesses'] = df['p_accesses']\n",
    "\n",
    "f['p_accesses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bytes\n",
    "f['p_bytes'] = df['posix_bytes_read'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_opens'] = df['posix_opens']\n",
    "f['p_seeks'] = df['posix_seeks']\n",
    "f['p_stats'] = df['posix_stats']\n",
    "f['p_mode'] = df['posix_renamed_mode']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['l_n_osts'] = df['lustre_number_of_osts']\n",
    "f['l_stripe_w'] = df['lustre_stripe_width']\n",
    "f['l_mdts'] = df['lustre_mdts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_nprocs'] = df['nprocs']\n",
    "f['p_falign'] = df['posix_f_align']\n",
    "f['p_malign'] = df['posix_m_align']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_reads'] = df['posix_reads'] \n",
    "f['p_writes'] = df['posix_writes'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_bytes_read_100'] = df['posix_bytes_read_100']\n",
    "f['p_bytes_read_1K'] = df['posix_bytes_read_1K']\n",
    "f['p_bytes_read_10K'] = df['posix_bytes_read_10K'] \n",
    "f['p_bytes_read_100K'] = df['posix_bytes_read_100K']\n",
    "f['p_bytes_read_1M'] = df['posix_bytes_read_1lM']\n",
    "f['p_bytes_read_4M'] = df['posix_bytes_read_4M']\n",
    "f['p_bytes_read_10M'] = df['posix_bytes_read_10M']\n",
    "f['p_bytes_read_100M'] = df['posix_bytes_read_100M']\n",
    "f['p_bytes_read_1G'] = df['posix_bytes_read_1G']\n",
    "f['p_bytes_read_PLUS'] = df['posix_bytes_read_PLUS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_bytes_write_100'] = df['posix_bytes_write_100']\n",
    "f['p_bytes_write_1K'] = df['posix_bytes_write_1K']\n",
    "f['p_bytes_write_10K'] = df['posix_bytes_write_10K']\n",
    "f['p_bytes_write_100K'] = df['posix_bytes_write_100K']\n",
    "f['p_bytes_write_1M'] = df['posix_bytes_write_1M']\n",
    "f['p_bytes_write_4M'] = df['posix_bytes_write_4M']\n",
    "f['p_bytes_write_10M'] = df['posix_bytes_write_10M']\n",
    "f['p_bytes_write_100M'] = df['posix_bytes_write_100M']\n",
    "f['p_bytes_write_1G'] = df['posix_bytes_write_1G']\n",
    "f['p_bytes_write_PLUS'] = df['posix_bytes_write_PLUS']\n",
    "\n",
    "\n",
    "f = f.replace(-np.inf, -1)\n",
    "f = f.replace(np.nan, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['time'] = df['posix_write_time'].astype('float') + df['posix_read_time'].astype('float') + df['posix_meta_time'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['bytes'] = df['posix_bytes_read'].astype('float') + df['posix_bytes_write'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df[df['bytes'] >99999999]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "f['throughput'] = df['bytes'].astype('float') / df['time']\n",
    "\n",
    "f = f[f['throughput'] >0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete columns with all zeros\n",
    "f = f.loc[:, (f != 0).any(axis=0)]\n",
    "\n",
    "#remove infinite values\n",
    "f = f.replace([np.inf, -np.inf], np.nan).dropna(axis=0)\n",
    "\n",
    "f.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.DataFrame()\n",
    "t['throughput'] = f['throughput']\n",
    "f = f.drop(labels = 'throughput', axis = 1)\n",
    "columns = f.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.index.isin(t.index)]\n",
    "t = t.reset_index()\n",
    "f = f.reset_index()\n",
    "f = f.drop(f.columns[0] , axis =1)\n",
    "t = t.drop(t.columns[0] , axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "f = StandardScaler().fit_transform(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(t.min())\n",
    "print(t.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rseed = 0\n",
    "t_size = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_absolute_percentage_error\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "for i in range(3):\n",
    "    \n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "\n",
    "    reg = LinearRegression().fit(train_data, train_labels)\n",
    "\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#EXTREME GRADIENT BOOST\n",
    "\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "    xgb_r = xg.XGBRegressor(n_estimators = 1000, seed = 123)\n",
    "    xgb_r.fit(train_data, train_labels)\n",
    "    predicted_labels = xgb_r.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "e = predicted_labels[0] - test_labels['throughput']\n",
    "e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.subplot(211)\n",
    "            \n",
    "plt.hist(predicted_labels, bins = 300)\n",
    "plt.grid(visible='on')\n",
    "plt.xlim([0,2.5*10**9])\n",
    "plt.yscale('log')\n",
    "plt.title('Model Predictions (top) vs. Observed Values (bottom)')\n",
    "\n",
    "plt.subplot(212)\n",
    "\n",
    "plt.ylabel('Number of Observations')\n",
    "plt.xlabel('Throughput Bytes/s')\n",
    "\n",
    "plt.hist(test_labels, bins = 300)\n",
    "plt.grid(visible='on')\n",
    "plt.xlim([0,2.5*10**9])\n",
    "plt.yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(e, bins = 1000)\n",
    "plt.grid(visible='on')\n",
    "#plt.xlim([-0.25*10**9,0.25*10**9])\n",
    "#plt.ylim(0,35000)\n",
    "\n",
    "plt.grid(visible='on')\n",
    "plt.title('Error Historgram, XGB')\n",
    "plt.xlabel('Error, Bytes/s')\n",
    "plt.ylabel('Number of Occurrances')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(columns,xgb_r.feature_importances_)\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "plt.grid(visible='on')\n",
    "plt.title('Feature Importances of the  Features in XGB Global Model')\n",
    "\n",
    "plt.ylabel('Feature Importance Score')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size= t_size, random_state=rseed)\n",
    "    \n",
    "    reg = DecisionTreeRegressor(max_depth = 4)   \n",
    "    \n",
    "\n",
    "    reg.fit(train_data, train_labels)\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}