udacity-project.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from azureml.core import Workspace, Experiment\n",
        "\n",
        "# Refer to Ishmeet S reply to the question: \n",
        "# https://knowledge.udacity.com/questions/371486\n",
        "# ws = Workspace.get(name=\"udacity-project\")--> This can only work if I create a workspace named \"udacity-project\" in the Azure ML studio .\n",
        "ws = Workspace.from_config()\n",
        "exp = Experiment(workspace=ws, name=\"udacity-project\")\n",
        "\n",
        "print('Workspace name: ' + ws.name, \n",
        "      'Azure region: ' + ws.location, \n",
        "      'Subscription id: ' + ws.subscription_id, \n",
        "      'Resource group: ' + ws.resource_group, sep = '\\n')\n",
        "\n",
        "run = exp.start_logging()\n",
        "print(ws)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Workspace name: quick-starts-ws-131112\n",
            "Azure region: southcentralus\n",
            "Subscription id: a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\n",
            "Resource group: aml-quickstarts-131112\n",
            "Workspace.create(name='quick-starts-ws-131112', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-131112')\n"
          ]
        }
      ],
      "execution_count": 15,
      "metadata": {
        "gather": {
          "logged": 1608467191232
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
        "from azureml.core.compute_target import ComputeTargetException\n",
        "\n",
        "# TODO: Create compute cluster\n",
        "# Use vm_size = \"Standard_D2_V2\" in your provisioning configuration.\n",
        "# max_nodes should be no greater than 4.\n",
        "\n",
        "# Choose a name for the cluster\n",
        "cpu_cluster_name = \"compute-cluster\"\n",
        "\n",
        "# Verify that cluster does not exist already\n",
        "try:\n",
        "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
        "    print('Found existing cluster, use it.')\n",
        "except ComputeTargetException:\n",
        "    print('Creating a new compute cluster...')\n",
        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', max_nodes=4)\n",
        "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
        "\n",
        "# Can poll for a minimum number of nodes and for a specific timeout. \n",
        "# If no min node count is provided it uses the scale settings for the cluster.\n",
        "compute_target.wait_for_completion(show_output=True)\n",
        "\n",
        "# use get_status() to get a detailed status for the current cluster. \n",
        "print(compute_target.get_status().serialize())\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating a new compute cluster...\n",
            "Creating\n",
            "Succeeded\n",
            "AmlCompute wait for completion finished\n",
            "\n",
            "Minimum number of nodes requested have been provisioned\n",
            "{'currentNodeCount': 0, 'targetNodeCount': 0, 'nodeStateCounts': {'preparingNodeCount': 0, 'runningNodeCount': 0, 'idleNodeCount': 0, 'unusableNodeCount': 0, 'leavingNodeCount': 0, 'preemptedNodeCount': 0}, 'allocationState': 'Steady', 'allocationStateTransitionTime': '2020-12-20T12:26:41.190000+00:00', 'errors': None, 'creationTime': '2020-12-20T12:26:38.177568+00:00', 'modifiedTime': '2020-12-20T12:26:53.948054+00:00', 'provisioningState': 'Succeeded', 'provisioningStateTransitionTime': None, 'scaleSettings': {'minNodeCount': 0, 'maxNodeCount': 4, 'nodeIdleTimeBeforeScaleDown': 'PT120S'}, 'vmPriority': 'Dedicated', 'vmSize': 'STANDARD_D2_V2'}\n"
          ]
        }
      ],
      "execution_count": 16,
      "metadata": {
        "gather": {
          "logged": 1608467212734
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from azureml.widgets import RunDetails\n",
        "from azureml.train.sklearn import SKLearn\n",
        "from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
        "from azureml.train.hyperdrive.policy import BanditPolicy\n",
        "from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
        "from azureml.train.hyperdrive.runconfig import HyperDriveConfig\n",
        "from azureml.train.hyperdrive.parameter_expressions import choice, uniform\n",
        "import os\n",
        "\n",
        "# Specify parameter sampler\n",
        "\n",
        "ps = RandomParameterSampling(\n",
        "    {\n",
        "        '--C' : choice(0.001,0.01,0.1,1,10,20,50,100,200,500,1000),\n",
        "        '--max_iter': choice(50,100,200,300)\n",
        "    }\n",
        ")\n",
        "\n",
        "# Specify a Policy\n",
        "policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1)\n",
        "\n",
        "if \"training\" not in os.listdir():\n",
        "    os.mkdir(\"./training\")\n",
        "\n",
        "# Create a SKLearn estimator for use with train.py\n",
        "est = SKLearn(source_directory = \"./\",\n",
        "            compute_target=compute_target,\n",
        "            vm_size='STANDARD_D2_V2',\n",
        "            entry_script=\"train.py\")\n",
        "\n",
        "# Create a HyperDriveConfig using the estimator, hyperparameter sampler, and policy.\n",
        "hyperdrive_config = HyperDriveConfig(hyperparameter_sampling=ps, \n",
        "                                     primary_metric_name='Accuracy',\n",
        "                                     primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
        "                                     policy=policy,\n",
        "                                     estimator=est,\n",
        "                                     max_total_runs=16,\n",
        "                                     max_concurrent_runs=4)\n",
        "\n"
      ],
      "outputs": [],
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1608466016239
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Submit your hyperdrive run to the experiment and show run details with the widget.\n",
        "\n",
        "# Start the HyperDrive run\n",
        "hyperdrive_run = exp.submit(hyperdrive_config)\n",
        "\n",
        "# Monitor HyperDrive runs You can monitor the progress of the runs with the following Jupyter widget\n",
        "# RunDetails(hyperdrive_run).show()\n",
        "\n",
        "hyperdrive_run.wait_for_completion(show_output=True)\n",
        "\n",
        "assert(hyperdrive_run.get_status() == \"Completed\")\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:If 'script' has been provided here and a script file name has been specified in 'run_config', 'script' provided in ScriptRunConfig initialization will take precedence.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RunId: HD_fda34223-a94c-456b-8bf7-52e84aa1d17e\n",
            "Web View: https://ml.azure.com/experiments/udacity-project/runs/HD_fda34223-a94c-456b-8bf7-52e84aa1d17e?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-131112/workspaces/quick-starts-ws-131112\n",
            "\n",
            "Streaming azureml-logs/hyperdrive.txt\n",
            "=====================================\n",
            "\n",
            "\"<START>[2020-12-20T12:06:58.894473][API][INFO]Experiment created<END>\\n\"\"<START>[2020-12-20T12:06:59.402645][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space<END>\\n\"<START>[2020-12-20T12:07:00.0064856Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.<END>\"<START>[2020-12-20T12:06:59.575960][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.<END>\\n\"\n",
            "\n",
            "Execution Summary\n",
            "=================\n",
            "RunId: HD_fda34223-a94c-456b-8bf7-52e84aa1d17e\n",
            "Web View: https://ml.azure.com/experiments/udacity-project/runs/HD_fda34223-a94c-456b-8bf7-52e84aa1d17e?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-131112/workspaces/quick-starts-ws-131112\n",
            "\n"
          ]
        }
      ],
      "execution_count": 4,
      "metadata": {
        "gather": {
          "logged": 1608466752166
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "\n",
        "# get_children_sorted_by_primary_metric: Returns a list of children sorted by their best primary metric.\n",
        "# Each child in the result has run id, hyperparameters, best primary metric value and status.\n",
        "\n",
        "print(hyperdrive_run.get_children_sorted_by_primary_metric(top=0, reverse=False, discard_no_metric=False))\n",
        "\n",
        "# Get your best run and save the model from that run.\n",
        "\n",
        "# get_best_run_by_primary_metric()\n",
        "# Returns the best Run, or None if no child has the primary metric.\n",
        "best_run = hyperdrive_run.get_best_run_by_primary_metric()\n",
        "# get_metrics()\n",
        "# Returns the metrics from all the runs that were launched by this HyperDriveRun.\n",
        "print(\"Best run metrics :\",best_run.get_metrics())\n",
        "# get_details()\n",
        "# Returns a dictionary with the details for the run\n",
        "print(\"Best run details :\",best_run.get_details())\n",
        "# get_file_names()\n",
        "# Returns a list of the files that are stored in association with the run.\n",
        "print(\"Best run file names :\",best_run.get_file_names())\n",
        "# log_accuracy_table()\n",
        "# Logs an accuracy table to the artifact store. Maybe use it for testing.\n",
        "# log_accuracy_table(name, value, description='')\n",
        "\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14', 'hyperparameters': None, 'best_primary_metric': 0.9176024279210926, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_0', 'hyperparameters': None, 'best_primary_metric': 0.9165402124430956, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_2', 'hyperparameters': None, 'best_primary_metric': 0.9165402124430956, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_15', 'hyperparameters': None, 'best_primary_metric': 0.9163884673748103, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_7', 'hyperparameters': None, 'best_primary_metric': 0.9163884673748103, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_6', 'hyperparameters': None, 'best_primary_metric': 0.9163884673748103, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_4', 'hyperparameters': None, 'best_primary_metric': 0.9163884673748103, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_1', 'hyperparameters': None, 'best_primary_metric': 0.9163884673748103, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_13', 'hyperparameters': None, 'best_primary_metric': 0.9162367223065251, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_8', 'hyperparameters': None, 'best_primary_metric': 0.9162367223065251, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_9', 'hyperparameters': None, 'best_primary_metric': 0.9162367223065251, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_11', 'hyperparameters': None, 'best_primary_metric': 0.9159332321699545, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_10', 'hyperparameters': None, 'best_primary_metric': 0.9159332321699545, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_5', 'hyperparameters': None, 'best_primary_metric': 0.9159332321699545, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_12', 'hyperparameters': None, 'best_primary_metric': 0.9157814871016692, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_3', 'hyperparameters': None, 'best_primary_metric': 0.9157814871016692, 'status': 'Completed'}, {'run_id': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_preparation', 'hyperparameters': None, 'best_primary_metric': None, 'status': 'Completed'}]\n",
            "Best run metrics : {'Regularization Strength:': 100.0, 'Max iterations:': 100, 'Accuracy': 0.9176024279210926}\n",
            "Best run details : {'runId': 'HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14', 'target': 'compute-cluster', 'status': 'Completed', 'startTimeUtc': '2020-12-20T12:15:30.467589Z', 'endTimeUtc': '2020-12-20T12:17:37.167447Z', 'properties': {'_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': '24edd5b2-67c3-4b24-a903-393ec87008be', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json'}, 'inputDatasets': [], 'outputDatasets': [], 'runDefinition': {'script': 'train.py', 'useAbsolutePath': False, 'arguments': ['--C', '100', '--max_iter', '100'], 'sourceDirectoryDataStore': None, 'framework': 'Python', 'communicator': 'None', 'target': 'compute-cluster', 'dataReferences': {}, 'data': {}, 'outputData': {}, 'jobName': None, 'maxRunDurationSeconds': None, 'nodeCount': 1, 'priority': None, 'credentialPassthrough': False, 'environment': {'name': 'Experiment udacity-project Environment', 'version': 'Autosave_2020-12-20T11:11:27Z_44af389b', 'python': {'interpreterPath': 'python', 'userManagedDependencies': True, 'condaDependencies': {'name': 'project_environment', 'dependencies': ['python=3.6.2', {'pip': ['azureml-defaults']}], 'channels': ['anaconda', 'conda-forge']}, 'baseCondaEnvironment': None}, 'environmentVariables': {'EXAMPLE_ENV_VAR': 'EXAMPLE_VALUE'}, 'docker': {'baseImage': 'sklearn:0.20.3-cpu', 'platform': {'os': 'Linux', 'architecture': 'amd64'}, 'baseDockerfile': None, 'baseImageRegistry': {'address': 'viennaprivate.azurecr.io', 'username': None, 'password': None}, 'enabled': True, 'arguments': []}, 'spark': {'repositories': [], 'packages': [], 'precachePackages': False}, 'inferencingStackVersion': None}, 'history': {'outputCollection': True, 'directoriesToWatch': ['logs'], 'enableMLflowTracking': True, 'snapshotProject': True}, 'spark': {'configuration': {'spark.app.name': 'Azure ML Experiment', 'spark.yarn.maxAppAttempts': '1'}}, 'parallelTask': {'maxRetriesPerWorker': 0, 'workerCountPerNode': 1, 'terminalExitCodes': None, 'configuration': {}}, 'amlCompute': {'name': None, 'vmSize': 'STANDARD_D2_V2', 'retainCluster': False, 'clusterMaxNodeCount': 1}, 'aiSuperComputer': {'instanceType': None, 'frameworkImage': None, 'imageVersion': None, 'location': None, 'aiSuperComputerStorageData': None}, 'tensorflow': {'workerCount': 1, 'parameterServerCount': 1}, 'mpi': {'processCountPerNode': 1}, 'hdi': {'yarnDeployMode': 'Cluster'}, 'containerInstance': {'region': None, 'cpuCores': 2.0, 'memoryGb': 3.5}, 'exposedPorts': None, 'docker': {'useDocker': True, 'sharedVolumes': True, 'shmSize': '2g', 'arguments': []}, 'cmk8sCompute': {'configuration': {}}, 'commandReturnCodeConfig': {'returnCode': 'Zero', 'successfulReturnCodes': []}}, 'logFiles': {'azureml-logs/55_azureml-execution-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/55_azureml-execution-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt?sv=2019-02-02&sr=b&sig=ezthteKeTmeje9IJhLIW6zsTRihcPbw8Mpd4fIIR3k0%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'azureml-logs/65_job_prep-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/65_job_prep-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt?sv=2019-02-02&sr=b&sig=3n2vql0ivhx1%2B5Hzuqu90btLB44%2FXu3MtSHX38nfDvY%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'azureml-logs/70_driver_log.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/70_driver_log.txt?sv=2019-02-02&sr=b&sig=zkbkIzIkfAba9OraXe4RswjijpoT4LicKj9ksNhWMDA%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'azureml-logs/75_job_post-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/75_job_post-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt?sv=2019-02-02&sr=b&sig=32WbztoXm3VecpJh32LL3z3XTZUymgx6i8idw33OUKg%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'azureml-logs/process_info.json': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/process_info.json?sv=2019-02-02&sr=b&sig=VGvX3oksnqJLXbvn0EbyNonPsfiyE3GTfmcWPrlsOJQ%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'azureml-logs/process_status.json': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/azureml-logs/process_status.json?sv=2019-02-02&sr=b&sig=fcrJT5Cx8GLFzs6kyhdKrth248fDvptBlwa46TGaf0w%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'logs/azureml/101_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/logs/azureml/101_azureml.log?sv=2019-02-02&sr=b&sig=OjFNvqmw7c%2Fue3%2BSBG2oYf0sUl5kKUBQROUwPS6fMNU%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'logs/azureml/job_prep_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/logs/azureml/job_prep_azureml.log?sv=2019-02-02&sr=b&sig=tg91jngcwiOmwsz41CMdMRrM1drT6ojmrVdzof32Mag%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r', 'logs/azureml/job_release_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_fda34223-a94c-456b-8bf7-52e84aa1d17e_14/logs/azureml/job_release_azureml.log?sv=2019-02-02&sr=b&sig=cbnq9ah3gHqJ2fVGoH43TgXMsFD5ivHIm5xn9uwM3xg%3D&st=2020-12-20T12%3A09%3A17Z&se=2020-12-20T20%3A19%3A17Z&sp=r'}}\n",
            "Best run file names : ['azureml-logs/55_azureml-execution-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt', 'azureml-logs/65_job_prep-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt', 'azureml-logs/70_driver_log.txt', 'azureml-logs/75_job_post-tvmps_90c7ec669ccab40e4e054c3a9da0660bf864e36ff1a27584defc4d1a33030e82_d.txt', 'azureml-logs/process_info.json', 'azureml-logs/process_status.json', 'logs/azureml/101_azureml.log', 'logs/azureml/job_prep_azureml.log', 'logs/azureml/job_release_azureml.log']\n"
          ]
        }
      ],
      "execution_count": 5,
      "metadata": {
        "gather": {
          "logged": 1608466755917
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
        "\n",
        "# Create TabularDataset using TabularDatasetFactory\n",
        "# Data is available at: \n",
        "# \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
        "\n",
        "ds = TabularDatasetFactory.from_delimited_files(['https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv'])"
      ],
      "outputs": [],
      "execution_count": 17,
      "metadata": {
        "gather": {
          "logged": 1608467244524
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from train import clean_data\n",
        "\n",
        "# Use the clean_data function to clean your data.\n",
        "x, y = clean_data(ds)"
      ],
      "outputs": [],
      "execution_count": 18,
      "metadata": {
        "gather": {
          "logged": 1608467246686
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.train.automl import AutoMLConfig\n",
        "\n",
        "# Set parameters for AutoMLConfig \n",
        "# https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig?view=azure-ml-py\n",
        "\n",
        "# NOTE: DO NOT CHANGE THE experiment_timeout_minutes PARAMETER OR YOUR INSTANCE WILL TIME OUT.\n",
        "# If you wish to run the experiment longer, you will need to run this notebook in your own\n",
        "# Azure tenant, which will incur personal costs.\n",
        "\n",
        "automl_config = AutoMLConfig(\n",
        "    compute_target = compute_target,\n",
        "    experiment_timeout_minutes=15,\n",
        "    task='classification',\n",
        "    primary_metric='accuracy',\n",
        "    training_data=ds,\n",
        "    label_column_name='y',\n",
        "    enable_onnx_compatible_models=True,\n",
        "    n_cross_validations=2)\n"
      ],
      "outputs": [],
      "execution_count": 19,
      "metadata": {
        "gather": {
          "logged": 1608467249154
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Submit your automl run\n",
        "\n",
        "remote_run = exp.submit(automl_config, show_output = False)\n",
        "remote_run.wait_for_completion()"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on remote.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "execution_count": 20,
          "data": {
            "text/plain": "{'runId': 'AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c',\n 'target': 'compute-cluster',\n 'status': 'Completed',\n 'startTimeUtc': '2020-12-20T12:27:58.224053Z',\n 'endTimeUtc': '2020-12-20T12:53:07.771424Z',\n 'properties': {'num_iterations': '1000',\n  'training_type': 'TrainFull',\n  'acquisition_function': 'EI',\n  'primary_metric': 'accuracy',\n  'train_split': '0',\n  'acquisition_parameter': '0',\n  'num_cross_validation': '2',\n  'target': 'compute-cluster',\n  'AMLSettingsJsonString': '{\"path\":null,\"name\":\"udacity-project\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group\":\"aml-quickstarts-131112\",\"workspace_name\":\"quick-starts-ws-131112\",\"region\":\"southcentralus\",\"compute_target\":\"compute-cluster\",\"spark_service\":null,\"azure_service\":\"remote\",\"_local_managed_run_id\":null,\"many_models\":false,\"pipeline_fetch_max_batch_size\":1,\"iterations\":1000,\"primary_metric\":\"accuracy\",\"task_type\":\"classification\",\"data_script\":null,\"validation_size\":0.0,\"n_cross_validations\":2,\"y_min\":null,\"y_max\":null,\"num_classes\":null,\"featurization\":\"auto\",\"_ignore_package_version_incompatibilities\":false,\"is_timeseries\":false,\"max_cores_per_iteration\":1,\"max_concurrent_iterations\":1,\"iteration_timeout_minutes\":null,\"mem_in_mb\":null,\"enforce_time_on_windows\":false,\"experiment_timeout_minutes\":15,\"experiment_exit_score\":null,\"whitelist_models\":null,\"blacklist_algos\":[\"TensorFlowLinearClassifier\",\"TensorFlowDNN\"],\"supported_models\":[\"LinearSVM\",\"SVM\",\"AveragedPerceptronClassifier\",\"TensorFlowLinearClassifier\",\"KNN\",\"LogisticRegression\",\"DecisionTree\",\"RandomForest\",\"ExtremeRandomTrees\",\"TensorFlowDNN\",\"SGD\",\"BernoulliNaiveBayes\",\"GradientBoosting\",\"MultinomialNaiveBayes\",\"LightGBM\",\"XGBoostClassifier\"],\"auto_blacklist\":true,\"blacklist_samples_reached\":false,\"exclude_nan_labels\":true,\"verbosity\":20,\"_debug_log\":\"azureml_automl.log\",\"show_warnings\":false,\"model_explainability\":true,\"service_url\":null,\"sdk_url\":null,\"sdk_packages\":null,\"enable_onnx_compatible_models\":true,\"enable_split_onnx_featurizer_estimator_models\":false,\"vm_type\":\"STANDARD_D2_V2\",\"telemetry_verbosity\":20,\"send_telemetry\":true,\"enable_dnn\":false,\"scenario\":\"SDK-1.13.0\",\"environment_label\":null,\"force_text_dnn\":false,\"enable_feature_sweeping\":false,\"enable_early_stopping\":false,\"early_stopping_n_iters\":10,\"metrics\":null,\"enable_ensembling\":true,\"enable_stack_ensembling\":false,\"ensemble_iterations\":15,\"enable_tf\":false,\"enable_subsampling\":null,\"subsample_seed\":null,\"enable_nimbusml\":false,\"enable_streaming\":false,\"force_streaming\":false,\"track_child_runs\":true,\"allowed_private_models\":[],\"label_column_name\":\"y\",\"weight_column_name\":null,\"cv_split_column_names\":null,\"enable_local_managed\":false,\"cost_mode\":1,\"lag_length\":0,\"metric_operation\":\"maximize\",\"preprocess\":true}',\n  'DataPrepJsonString': '{\\\\\"training_data\\\\\": \\\\\"{\\\\\\\\\\\\\"blocks\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"b85085ba-e466-46a4-837d-d1bb9245128c\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.GetFilesBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"isArchive\\\\\\\\\\\\\": false, \\\\\\\\\\\\\"path\\\\\\\\\\\\\": {\\\\\\\\\\\\\"target\\\\\\\\\\\\\": 1, \\\\\\\\\\\\\"resourceDetails\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"path\\\\\\\\\\\\\": \\\\\\\\\\\\\"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\\\\\\\\\\\\\", \\\\\\\\\\\\\"sas\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"storageAccountName\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"storageAccountKey\\\\\\\\\\\\\": null}]}}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"ebb272ef-1139-4f96-a392-f202086015d5\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.ParseDelimitedBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columnHeadersMode\\\\\\\\\\\\\": 3, \\\\\\\\\\\\\"fileEncoding\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"handleQuotedLineBreaks\\\\\\\\\\\\\": false, \\\\\\\\\\\\\"preview\\\\\\\\\\\\\": false, \\\\\\\\\\\\\"separator\\\\\\\\\\\\\": \\\\\\\\\\\\\",\\\\\\\\\\\\\", \\\\\\\\\\\\\"skipRows\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"skipRowsMode\\\\\\\\\\\\\": 0}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"2f8eb952-b796-4c76-9bb9-132616818b7a\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.DropColumnsBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columns\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumns\\\\\\\\\\\\\": [\\\\\\\\\\\\\"Path\\\\\\\\\\\\\"]}}}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"3bfa7c70-c188-41c9-adae-0bf94f2fdde3\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.SetColumnTypesBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columnConversion\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"age\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"job\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"marital\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"education\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"default\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"housing\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"loan\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"contact\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"month\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"day_of_week\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"duration\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"campaign\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"pdays\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"previous\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"poutcome\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"emp.var.rate\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 3}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"cons.price.idx\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 3}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"cons.conf.idx\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 3}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"euribor3m\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 3}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"nr.employed\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 3}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"y\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}]}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}], \\\\\\\\\\\\\"inspectors\\\\\\\\\\\\\": [], \\\\\\\\\\\\\"meta\\\\\\\\\\\\\": {\\\\\\\\\\\\\"savedDatasetId\\\\\\\\\\\\\": \\\\\\\\\\\\\"8789ed4b-cbb2-4822-aae9-ccbbf5a5f973\\\\\\\\\\\\\", \\\\\\\\\\\\\"datasetType\\\\\\\\\\\\\": \\\\\\\\\\\\\"tabular\\\\\\\\\\\\\", \\\\\\\\\\\\\"subscriptionId\\\\\\\\\\\\\": \\\\\\\\\\\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\\\\\\\\\\\", \\\\\\\\\\\\\"workspaceId\\\\\\\\\\\\\": \\\\\\\\\\\\\"287c68f0-6fd3-4ce9-8033-d1fc62c30610\\\\\\\\\\\\\", \\\\\\\\\\\\\"workspaceLocation\\\\\\\\\\\\\": \\\\\\\\\\\\\"southcentralus\\\\\\\\\\\\\"}}\\\\\", \\\\\"activities\\\\\": 0}',\n  'EnableSubsampling': None,\n  'runTemplate': 'AutoML',\n  'azureml.runsource': 'automl',\n  'display_task_type': 'classification',\n  'dependencies_versions': '{\"azureml-train\": \"1.18.0\", \"azureml-train-restclients-hyperdrive\": \"1.18.0\", \"azureml-train-core\": \"1.18.0.post1\", \"azureml-train-automl\": \"1.18.0\", \"azureml-train-automl-runtime\": \"1.18.0.post1\", \"azureml-train-automl-client\": \"1.18.0\", \"azureml-telemetry\": \"1.18.0.post1\", \"azureml-sdk\": \"1.18.0\", \"azureml-samples\": \"0+unknown\", \"azureml-pipeline\": \"1.18.0\", \"azureml-pipeline-steps\": \"1.18.0\", \"azureml-pipeline-core\": \"1.18.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-interpret\": \"1.18.0\", \"azureml-defaults\": \"1.18.0\", \"azureml-dataset-runtime\": \"1.18.0\", \"azureml-dataprep\": \"2.4.5\", \"azureml-dataprep-rslex\": \"1.2.3\", \"azureml-dataprep-native\": \"24.0.0\", \"azureml-core\": \"1.18.0.post4\", \"azureml-contrib-services\": \"1.18.0\", \"azureml-contrib-server\": \"1.18.0\", \"azureml-contrib-pipeline-steps\": \"1.18.0\", \"azureml-contrib-notebook\": \"1.18.0\", \"azureml-automl-runtime\": \"1.18.0.post2\", \"azureml-automl-core\": \"1.18.0.post2\"}',\n  '_aml_system_scenario_identification': 'Remote.Parent',\n  'ClientType': 'SDK',\n  'environment_cpu_name': 'AzureML-AutoML',\n  'environment_cpu_version': '49',\n  'environment_gpu_name': 'AzureML-AutoML-GPU',\n  'environment_gpu_version': '37',\n  'root_attribution': 'automl',\n  'attribution': 'AutoML',\n  'Orchestrator': 'AutoML',\n  'CancelUri': 'https://southcentralus.experiments.azureml.net/jasmine/v1.0/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-131112/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-131112/experimentids/625c6990-2e9b-4bba-809b-92590b03e475/cancel/AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c',\n  'ClientSdkVersion': '1.18.0.post1',\n  'snapshotId': '00000000-0000-0000-0000-000000000000',\n  'SetupRunId': 'AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_setup',\n  'SetupRunContainerId': 'dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_setup',\n  'FeaturizationRunJsonPath': 'featurizer_container.json',\n  'FeaturizationRunId': 'AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_featurize',\n  'ProblemInfoJsonString': '{\"dataset_num_categorical\": 0, \"is_sparse\": true, \"subsampling\": false, \"dataset_classes\": 2, \"dataset_features\": 132, \"dataset_samples\": 32950, \"single_frequency_class_detected\": false}',\n  'ModelExplainRunId': 'AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_ModelExplain'},\n 'inputDatasets': [{'dataset': {'id': '8789ed4b-cbb2-4822-aae9-ccbbf5a5f973'}, 'consumptionDetails': {'type': 'RunInput', 'inputName': 'training_data', 'mechanism': 'Direct'}}],\n 'outputDatasets': [],\n 'logFiles': {}}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 20,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1608468790035
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Retrieve and save your best automl model.\n",
        "\n",
        "best_run, fitted_model = remote_run.get_output()\n",
        "\n",
        "print(best_run)\n",
        "print(fitted_model)\n",
        "\n",
        "# get_metrics()\n",
        "# Returns the metrics\n",
        "print(\"Best run metrics :\",best_run.get_metrics())\n",
        "# get_details()\n",
        "# Returns a dictionary with the details for the run\n",
        "print(\"Best run details :\",best_run.get_details())\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Run(Experiment: udacity-project,\n",
            "Id: AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13,\n",
            "Type: azureml.scriptrun,\n",
            "Status: Completed)\n",
            "Pipeline(memory=None,\n",
            "         steps=[('datatransformer',\n",
            "                 DataTransformer(enable_dnn=None, enable_feature_sweeping=None,\n",
            "                                 feature_sweeping_config=None,\n",
            "                                 feature_sweeping_timeout=None,\n",
            "                                 featurization_config=None, force_text_dnn=None,\n",
            "                                 is_cross_validation=None,\n",
            "                                 is_onnx_compatible=None, logger=None,\n",
            "                                 observer=None, task=None, working_dir=None)),\n",
            "                ('prefittedsoftvotingclassifier',...\n",
            "                                                                                                    min_impurity_split=None,\n",
            "                                                                                                    min_samples_leaf=0.01,\n",
            "                                                                                                    min_samples_split=0.01,\n",
            "                                                                                                    min_weight_fraction_leaf=0.0,\n",
            "                                                                                                    n_estimators=25,\n",
            "                                                                                                    n_jobs=1,\n",
            "                                                                                                    oob_score=True,\n",
            "                                                                                                    random_state=None,\n",
            "                                                                                                    verbose=0,\n",
            "                                                                                                    warm_start=False))],\n",
            "                                                                     verbose=False))],\n",
            "                                               flatten_transform=None,\n",
            "                                               weights=[0.14285714285714285,\n",
            "                                                        0.42857142857142855,\n",
            "                                                        0.14285714285714285,\n",
            "                                                        0.07142857142857142,\n",
            "                                                        0.14285714285714285,\n",
            "                                                        0.07142857142857142]))],\n",
            "         verbose=False)\n",
            "Y_transformer(['LabelEncoder', LabelEncoder()])\n",
            "Best run metrics : {'recall_score_weighted': 0.916176024279211, 'precision_score_micro': 0.916176024279211, 'accuracy': 0.916176024279211, 'f1_score_macro': 0.7670888929822944, 'average_precision_score_weighted': 0.9551269813011192, 'balanced_accuracy': 0.7422704472987491, 'f1_score_micro': 0.916176024279211, 'log_loss': 0.20352616228137643, 'recall_score_macro': 0.7422704472987491, 'norm_macro_recall': 0.4845408945974984, 'average_precision_score_macro': 0.8239229210674028, 'precision_score_macro': 0.8008848990333693, 'recall_score_micro': 0.916176024279211, 'average_precision_score_micro': 0.9813297644870336, 'AUC_micro': 0.9805953509363752, 'AUC_weighted': 0.9469939634729121, 'precision_score_weighted': 0.9094507932912679, 'matthews_correlation': 0.5399240901530455, 'AUC_macro': 0.9469939634729121, 'weighted_accuracy': 0.9593764379515999, 'f1_score_weighted': 0.9116727219350089, 'confusion_matrix': 'aml://artifactId/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/confusion_matrix', 'accuracy_table': 'aml://artifactId/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/accuracy_table'}\n",
            "Best run details : {'runId': 'AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13', 'target': 'compute-cluster', 'status': 'Completed', 'startTimeUtc': '2020-12-20T12:51:52.320396Z', 'endTimeUtc': '2020-12-20T12:52:58.399211Z', 'properties': {'runTemplate': 'automl_child', 'pipeline_id': '__AutoML_Ensemble__', 'pipeline_spec': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'accuracy\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'udacity-project\\',\\'compute_target\\':\\'compute-cluster\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13\",\"experiment_name\":\"udacity-project\",\"workspace_name\":\"quick-starts-ws-131112\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-131112\"}}]}', 'training_percent': '100', 'predicted_cost': None, 'iteration': '13', '_aml_system_scenario_identification': 'Remote.Child', '_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': '106e4190-841e-4961-805b-84e36a47edca', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json', 'run_template': 'automl_child', 'run_preprocessor': '', 'run_algorithm': 'VotingEnsemble', 'conda_env_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/outputs/conda_env_v_1_0_0.yml', 'model_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/outputs/model.pkl', 'model_size_on_disk': '1884139', 'scoring_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/outputs/scoring_file_v_1_0_0.py', 'model_exp_support': 'True', 'pipeline_graph_version': '1.0.0', 'model_name': 'AutoMLee4a685e313', 'staticProperties': '{}', 'score': '0.916176024279211', 'run_properties': \"classification_labels=None,\\n                              estimators=[('1',\\n                                           Pipeline(memory=None,\\n                                                    steps=[('maxabsscaler',\\n                                                            MaxAbsScaler(copy=True\", 'pipeline_script': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'accuracy\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'udacity-project\\',\\'compute_target\\':\\'compute-cluster\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13\",\"experiment_name\":\"udacity-project\",\"workspace_name\":\"quick-starts-ws-131112\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-131112\"}}]}', 'training_type': 'MeanCrossValidation', 'num_classes': '', 'framework': 'sklearn', 'fit_time': '18', 'goal': 'accuracy_max', 'class_labels': '', 'primary_metric': 'accuracy', 'errors': '{}', 'fitted_pipeline': \"Pipeline(memory=None,\\n         steps=[('datatransformer',\\n                 DataTransformer(enable_dnn=None, enable_feature_sweeping=None,\\n                                 feature_sweeping_config=None,\\n                                 feature_sweeping_timeout=None,\\n                                 featurization_config=None, force_text_dnn=None,\\n                                 is_cross_validation=None,\\n                                 is_onnx_compatible=None, logger=None,\\n                                 observer=None, task=None, working_dir=None)),\\n                ('prefittedsoftvotingclassifier',...\\n                                                                                                    min_impurity_split=None,\\n                                                                                                    min_samples_leaf=0.01,\\n                                                                                                    min_samples_split=0.01,\\n                                                                                                    min_weight_fraction_leaf=0.0,\\n                                                                                                    n_estimators=25,\\n                                                                                                    n_jobs=1,\\n                                                                                                    oob_score=True,\\n                                                                                                    random_state=None,\\n                                                                                                    verbose=0,\\n                                                                                                    warm_start=False))],\\n                                                                     verbose=False))],\\n                                               flatten_transform=None,\\n                                               weights=[0.14285714285714285,\\n                                                        0.42857142857142855,\\n                                                        0.14285714285714285,\\n                                                        0.07142857142857142,\\n                                                        0.14285714285714285,\\n                                                        0.07142857142857142]))],\\n         verbose=False)\\nY_transformer(['LabelEncoder', LabelEncoder()])\", 'friendly_errors': '{}', 'onnx_model_resource': \"{'RawColumnNameToOnnxNameMap': {'age': 'age', 'job': 'job', 'marital': 'marital', 'education': 'education', 'default': 'default', 'housing': 'housing', 'loan': 'loan', 'contact': 'contact', 'month': 'month', 'day_of_week': 'day_of_week', 'duration': 'duration', 'campaign': 'campaign', 'pdays': 'pdays', 'previous': 'previous', 'poutcome': 'poutcome', 'emp.var.rate': 'emp_var_rate', 'cons.price.idx': 'cons_price_idx', 'cons.conf.idx': 'cons_conf_idx', 'euribor3m': 'euribor3m', 'nr.employed': 'nr_employed'}, 'InputRawColumnSchema': {'age': 'integer', 'job': 'string', 'marital': 'string', 'education': 'string', 'default': 'string', 'housing': 'string', 'loan': 'string', 'contact': 'string', 'month': 'string', 'day_of_week': 'string', 'duration': 'integer', 'campaign': 'integer', 'pdays': 'integer', 'previous': 'integer', 'poutcome': 'string', 'emp.var.rate': 'floating', 'cons.price.idx': 'floating', 'cons.conf.idx': 'floating', 'euribor3m': 'floating', 'nr.employed': 'floating'}, 'InputOnnxColumnSchema': {'age': 'Int64TensorType', 'job': 'StringTensorType', 'marital': 'StringTensorType', 'education': 'StringTensorType', 'default': 'StringTensorType', 'housing': 'StringTensorType', 'loan': 'StringTensorType', 'contact': 'StringTensorType', 'month': 'StringTensorType', 'day_of_week': 'StringTensorType', 'duration': 'Int64TensorType', 'campaign': 'Int64TensorType', 'pdays': 'Int64TensorType', 'previous': 'Int64TensorType', 'poutcome': 'StringTensorType', 'emp_var_rate': 'FloatTensorType', 'cons_price_idx': 'FloatTensorType', 'cons_conf_idx': 'FloatTensorType', 'euribor3m': 'FloatTensorType', 'nr_employed': 'FloatTensorType'}, 'CanEnableBatchMode': True}\", 'error_code': '', 'failure_reason': '', 'feature_skus': 'automatedml_sdk_guardrails', 'dependencies_versions': '{\"azureml-train-automl-runtime\": \"1.18.0.post1\", \"azureml-train-automl-client\": \"1.18.0\", \"azureml-telemetry\": \"1.18.0.post1\", \"azureml-pipeline-core\": \"1.18.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-interpret\": \"1.18.0\", \"azureml-defaults\": \"1.18.0\", \"azureml-dataset-runtime\": \"1.18.0\", \"azureml-dataprep\": \"2.4.5\", \"azureml-dataprep-rslex\": \"1.2.3\", \"azureml-dataprep-native\": \"24.0.0\", \"azureml-core\": \"1.18.0.post4\", \"azureml-automl-runtime\": \"1.18.0.post2\", \"azureml-automl-core\": \"1.18.0.post2\"}', 'num_cores': '2', 'num_logical_cores': '2', 'peak_memory_usage': '521716', 'vm_configuration': 'Intel(R) Xeon(R) Platinum 8171M CPU @ 2.60GHz', 'core_hours': '0.004442403333333333'}, 'inputDatasets': [], 'outputDatasets': [], 'runDefinition': {'script': 'automl_driver.py', 'useAbsolutePath': False, 'arguments': [], 'sourceDirectoryDataStore': None, 'framework': 'Python', 'communicator': 'None', 'target': 'compute-cluster', 'dataReferences': {}, 'data': {}, 'outputData': {}, 'jobName': None, 'maxRunDurationSeconds': None, 'nodeCount': 1, 'priority': None, 'credentialPassthrough': False, 'environment': {'name': 'AutoML-AzureML-AutoML', 'version': 'Autosave_2020-12-20T11:25:06Z_6ce2c930', 'python': {'interpreterPath': 'python', 'userManagedDependencies': False, 'condaDependencies': {'channels': ['anaconda', 'conda-forge', 'pytorch'], 'dependencies': ['python=3.6.2', 'pip=20.2.4', {'pip': ['azureml-core==1.18.0.post4', 'azureml-pipeline-core==1.18.0', 'azureml-telemetry==1.18.0.post1', 'azureml-defaults==1.18.0', 'azureml-interpret==1.18.0', 'azureml-automl-core==1.18.0.post2', 'azureml-automl-runtime==1.18.0.post2', 'azureml-train-automl-client==1.18.0', 'azureml-train-automl-runtime==1.18.0.post1', 'azureml-dataset-runtime==1.18.0', 'inference-schema', 'py-cpuinfo==5.0.0', 'boto3==1.15.18', 'botocore==1.18.18']}, 'numpy~=1.18.0', 'scikit-learn==0.22.1', 'pandas~=0.25.0', 'py-xgboost<=0.90', 'fbprophet==0.5', 'holidays==0.9.11', 'setuptools-git', 'psutil>5.0.0,<6.0.0'], 'name': 'azureml_e1251fbb90896de9de2cde1255eba7bb'}, 'baseCondaEnvironment': None}, 'environmentVariables': {'EXAMPLE_ENV_VAR': 'EXAMPLE_VALUE'}, 'docker': {'baseImage': 'mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20201113.v1', 'platform': {'os': 'Linux', 'architecture': 'amd64'}, 'baseDockerfile': None, 'baseImageRegistry': {'address': None, 'username': None, 'password': None}, 'enabled': True, 'arguments': []}, 'spark': {'repositories': [], 'packages': [], 'precachePackages': True}, 'inferencingStackVersion': None}, 'history': {'outputCollection': True, 'directoriesToWatch': ['logs'], 'enableMLflowTracking': True}, 'spark': {'configuration': {'spark.app.name': 'Azure ML Experiment', 'spark.yarn.maxAppAttempts': '1'}}, 'parallelTask': {'maxRetriesPerWorker': 0, 'workerCountPerNode': 1, 'terminalExitCodes': None, 'configuration': {}}, 'amlCompute': {'name': None, 'vmSize': None, 'retainCluster': False, 'clusterMaxNodeCount': None}, 'aiSuperComputer': {'instanceType': None, 'frameworkImage': None, 'imageVersion': None, 'location': None, 'aiSuperComputerStorageData': None}, 'tensorflow': {'workerCount': 1, 'parameterServerCount': 1}, 'mpi': {'processCountPerNode': 1}, 'hdi': {'yarnDeployMode': 'Cluster'}, 'containerInstance': {'region': None, 'cpuCores': 2.0, 'memoryGb': 3.5}, 'exposedPorts': None, 'docker': {'useDocker': True, 'sharedVolumes': True, 'shmSize': '2g', 'arguments': []}, 'cmk8sCompute': {'configuration': {}}, 'commandReturnCodeConfig': {'returnCode': 'Zero', 'successfulReturnCodes': []}}, 'logFiles': {'azureml-logs/55_azureml-execution-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/55_azureml-execution-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt?sv=2019-02-02&sr=b&sig=tLI%2FUi8AN9YRpgwWgk0enApsb8MKNP4x%2FETM160%2FiwU%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'azureml-logs/65_job_prep-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/65_job_prep-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt?sv=2019-02-02&sr=b&sig=XZ49yKgdk0wqfKPajRnYJykwK4hJV9YT%2F6sLH49e32M%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'azureml-logs/70_driver_log.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/70_driver_log.txt?sv=2019-02-02&sr=b&sig=A5JSzIi8Qd6XujlwpA1dFCAwMjuOhBvlA%2Bt%2FPcUEO44%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'azureml-logs/75_job_post-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/75_job_post-tvmps_2ec5be13a5f707371e0f6caaa1b59290c2ad9f6fff86d74ae23f58aef28faa8b_d.txt?sv=2019-02-02&sr=b&sig=3ygmFfZx9UtDA1%2FTe2SWkDb3G0f%2BAmYuybRhv8RKO4U%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'azureml-logs/process_info.json': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/process_info.json?sv=2019-02-02&sr=b&sig=DyHhPV2BcAJqO1rywN2f0m%2BHTgkevQrhMVFFb2wKSpM%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'azureml-logs/process_status.json': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/azureml-logs/process_status.json?sv=2019-02-02&sr=b&sig=kvqgZY7AlCScHrGYNPWiC4%2F1X%2F%2BOAVIPgDoyptmMjAU%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/100_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/100_azureml.log?sv=2019-02-02&sr=b&sig=%2F%2FkD84D1DWIpRe%2Ft6uYEdT6S26yJBtuO01tIkGavVIk%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/azureml_automl.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/azureml_automl.log?sv=2019-02-02&sr=b&sig=uNCgTUgG4hmiWukFeKOs5l3yjJC0sczDDxJcAgFUh%2B0%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/dataprep/python_span_4dacc152-255a-4453-b174-2ff56c56fedd.jsonl': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/dataprep/python_span_4dacc152-255a-4453-b174-2ff56c56fedd.jsonl?sv=2019-02-02&sr=b&sig=qpq1M4a7b3UbpdqfBv6wJxw4J8j5DdC0XU7baIXsdXk%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/dataprep/python_span_80f7741b-4862-447c-b0f9-2b9ee95aa074.jsonl': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/dataprep/python_span_80f7741b-4862-447c-b0f9-2b9ee95aa074.jsonl?sv=2019-02-02&sr=b&sig=MHOV7f329v34C%2FseyJvn2NhTagpgPZls4v0SOkEzcjU%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/job_prep_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/job_prep_azureml.log?sv=2019-02-02&sr=b&sig=Z4obyvloKYb6YEh2f3KRwbMoT7delsYetE5d9KmrG9Y%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r', 'logs/azureml/job_release_azureml.log': 'https://mlstrg131112.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/logs/azureml/job_release_azureml.log?sv=2019-02-02&sr=b&sig=%2Bzg6AkwucosTrbGhhxWWjSE5KyTaA94sNJyv97fR61g%3D&st=2020-12-20T12%3A43%3A52Z&se=2020-12-20T20%3A53%3A52Z&sp=r'}}\n"
          ]
        }
      ],
      "execution_count": 21,
      "metadata": {
        "gather": {
          "logged": 1608468831008
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "best_run.get_metrics()"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 22,
          "data": {
            "text/plain": "{'recall_score_weighted': 0.916176024279211,\n 'precision_score_micro': 0.916176024279211,\n 'accuracy': 0.916176024279211,\n 'f1_score_macro': 0.7670888929822944,\n 'average_precision_score_weighted': 0.9551269813011192,\n 'balanced_accuracy': 0.7422704472987491,\n 'f1_score_micro': 0.916176024279211,\n 'log_loss': 0.20352616228137643,\n 'recall_score_macro': 0.7422704472987491,\n 'norm_macro_recall': 0.4845408945974984,\n 'average_precision_score_macro': 0.8239229210674028,\n 'precision_score_macro': 0.8008848990333693,\n 'recall_score_micro': 0.916176024279211,\n 'average_precision_score_micro': 0.9813297644870336,\n 'AUC_micro': 0.9805953509363752,\n 'AUC_weighted': 0.9469939634729121,\n 'precision_score_weighted': 0.9094507932912679,\n 'matthews_correlation': 0.5399240901530455,\n 'AUC_macro': 0.9469939634729121,\n 'weighted_accuracy': 0.9593764379515999,\n 'f1_score_weighted': 0.9116727219350089,\n 'confusion_matrix': 'aml://artifactId/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/confusion_matrix',\n 'accuracy_table': 'aml://artifactId/ExperimentRun/dcid.AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13/accuracy_table'}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 22,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1608468950783
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "fitted_model"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 23,
          "data": {
            "text/plain": "PipelineWithYTransformations(Pipeline={'memory': None,\n                                       'steps': [('datatransformer',\n                                                  DataTransformer(enable_dnn=None,\n                                                                  enable_feature_sweeping=None,\n                                                                  feature_sweeping_config=None,\n                                                                  feature_sweeping_timeout=None,\n                                                                  featurization_config=None,\n                                                                  force_text_dnn=None,\n                                                                  is_cross_validation=None,\n                                                                  is_onnx_compatible=None,\n                                                                  logger=None,\n                                                                  observer=None,\n                                                                  task=None,\n                                                                  working_dir=None))...\n                                                                                                                                     min_weight_fraction_leaf=0.0,\n                                                                                                                                     n_estimators=25,\n                                                                                                                                     n_jobs=1,\n                                                                                                                                     oob_score=True,\n                                                                                                                                     random_state=None,\n                                                                                                                                     verbose=0,\n                                                                                                                                     warm_start=False))],\n                                                                                                      verbose=False))],\n                                                                                flatten_transform=None,\n                                                                                weights=[0.14285714285714285,\n                                                                                         0.42857142857142855,\n                                                                                         0.14285714285714285,\n                                                                                         0.07142857142857142,\n                                                                                         0.14285714285714285,\n                                                                                         0.07142857142857142]))],\n                                       'verbose': False},\n                             y_transformer={},\n                             y_transformer_name='LabelEncoder')"
          },
          "metadata": {}
        }
      ],
      "execution_count": 23,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1608468967659
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "best_run"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 24,
          "data": {
            "text/plain": "Run(Experiment: udacity-project,\nId: AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13,\nType: azureml.scriptrun,\nStatus: Completed)",
            "text/html": "<table style=\"width:100%\"><tr><th>Experiment</th><th>Id</th><th>Type</th><th>Status</th><th>Details Page</th><th>Docs Page</th></tr><tr><td>udacity-project</td><td>AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13</td><td>azureml.scriptrun</td><td>Completed</td><td><a href=\"https://ml.azure.com/experiments/udacity-project/runs/AutoML_ee4a685e-34f2-4031-a4f9-fe96ff33836c_13?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-131112/workspaces/quick-starts-ws-131112\" target=\"_blank\" rel=\"noopener\">Link to Azure Machine Learning studio</a></td><td><a href=\"https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.run.Run?view=azure-ml-py\" target=\"_blank\" rel=\"noopener\">Link to Documentation</a></td></tr></table>"
          },
          "metadata": {}
        }
      ],
      "execution_count": 24,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1608468990767
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Cluster clean up\n",
        "\n",
        "compute_target.delete()"
      ],
      "outputs": [],
      "execution_count": 25,
      "metadata": {
        "gather": {
          "logged": 1608469075938
        }
      }
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python3"
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}