udacity-project.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from azureml.core import Workspace, Experiment\n",
        "\n",
        "ws = Workspace.from_config()\n",
        "# ws = Workspace.get(name=\"udacity-project\")\n",
        "exp = Experiment(workspace=ws, name=\"udacity-project\")\n",
        "\n",
        "print('Workspace name: ' + ws.name, \n",
        "      'Azure region: ' + ws.location, \n",
        "      'Subscription id: ' + ws.subscription_id, \n",
        "      'Resource group: ' + ws.resource_group, sep = '\\n')\n",
        "\n",
        "run = exp.start_logging()"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Workspace name: quick-starts-ws-170342\nAzure region: southcentralus\nSubscription id: 3d1a56d2-7c81-4118-9790-f85d1acf0c77\nResource group: aml-quickstarts-170342\n"
        }
      ],
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1640799206311
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
        "\n",
        "# TODO: Create compute cluster\n",
        "# Use vm_size = \"Standard_D2_V2\" in your provisioning configuration.\n",
        "# max_nodes should be no greater than 4.\n",
        "\n",
        "new_compute_cluster = 'project1'\n",
        "\n",
        "try:\n",
        "    compute_target = ComputeTarget(workspace=ws, name=new_compute_cluster)\n",
        "    print('Found existing compute cluster created via SDK: ', new_compute_cluster)\n",
        "except:\n",
        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', max_nodes=4)\n",
        "    compute_target = ComputeTarget.create(ws, new_compute_cluster, compute_config)\n",
        "    print('Created a new cluster via SDK.')\n",
        "\n",
        "compute_target.wait_for_completion(show_output=True)\n",
        "print(compute_target.get_status().serialize())"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Created a new cluster via SDK.\nInProgress..\nSucceededProvisioning operation finished, operation \"Succeeded\"\nSucceeded\nAmlCompute wait for completion finished\n\nMinimum number of nodes requested have been provisioned\n{'currentNodeCount': 0, 'targetNodeCount': 0, 'nodeStateCounts': {'preparingNodeCount': 0, 'runningNodeCount': 0, 'idleNodeCount': 0, 'unusableNodeCount': 0, 'leavingNodeCount': 0, 'preemptedNodeCount': 0}, 'allocationState': 'Resizing', 'allocationStateTransitionTime': '2021-12-29T17:33:27.094000+00:00', 'errors': None, 'creationTime': '2021-12-29T17:33:26.674648+00:00', 'modifiedTime': '2021-12-29T17:33:32.271364+00:00', 'provisioningState': 'Succeeded', 'provisioningStateTransitionTime': None, 'scaleSettings': {'minNodeCount': 0, 'maxNodeCount': 4, 'nodeIdleTimeBeforeScaleDown': 'PT1800S'}, 'vmPriority': 'Dedicated', 'vmSize': 'STANDARD_D2_V2'}\n"
        }
      ],
      "execution_count": 2,
      "metadata": {
        "gather": {
          "logged": 1640799217476
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.widgets import RunDetails\n",
        "from azureml.train.sklearn import SKLearn\n",
        "from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
        "from azureml.train.hyperdrive.policy import BanditPolicy\n",
        "from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
        "from azureml.train.hyperdrive.runconfig import HyperDriveConfig\n",
        "from azureml.train.hyperdrive.parameter_expressions import uniform, choice\n",
        "import os\n",
        "import shutil\n",
        "\n",
        "# Specify parameter sampler\n",
        "ps = RandomParameterSampling ({\n",
        "    \"--C\" : choice(0.01, 0.1, 1),\n",
        "    \"--max_iter\" : choice(20, 40, 60, 100, 150, 200) \n",
        "})\n",
        "\n",
        "# Specify a Policy\n",
        "policy = BanditPolicy(slack_factor=0.15, evaluation_interval=1, delay_evaluation=5)\n",
        "\n",
        "if \"training\" not in os.listdir():\n",
        "    os.mkdir(\"./training\")\n",
        "\n",
        "script_folder = \"./training\"\n",
        "os.makedirs(script_folder, exist_ok=True)\n",
        "shutil.copy(\"./train.py\", script_folder)\n",
        "\n",
        "# Create a SKLearn estimator for use with train.py\n",
        "est = SKLearn(\n",
        "    source_directory=script_folder,\n",
        "    compute_target=compute_target,\n",
        "    entry_script=\"train.py\",\n",
        "    vm_size=\"Standard_d2_v2\",\n",
        "    vm_priority=\"lowpriority\"\n",
        ")\n",
        "\n",
        "# Create a HyperDriveConfig using the estimator, hyperparameter sampler, and policy.\n",
        "hyperdrive_config = HyperDriveConfig(\n",
        "    hyperparameter_sampling=ps,\n",
        "    primary_metric_name=\"Accuracy\",\n",
        "    primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
        "    max_total_runs=4,\n",
        "    policy=policy,\n",
        "    estimator=est\n",
        ")"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "'SKLearn' estimator is deprecated. Please use 'ScriptRunConfig' from 'azureml.core.script_run_config' with your own defined environment or the AzureML-Tutorial curated environment.\n'enabled' is deprecated. Please use the azureml.core.runconfig.DockerConfiguration object with the 'use_docker' param instead.\n"
        }
      ],
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1640799224981
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Submit your hyperdrive run to the experiment and show run details with the widget.\n",
        "\n",
        "hyperdrive_run = exp.submit(config=hyperdrive_config)\n",
        "RunDetails(hyperdrive_run).show()\n",
        "\n",
        "hyperdrive_run.get_status()\n",
        "hyperdrive_run.wait_for_completion(show_output=True)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "WARNING:root:If 'script' has been provided here and a script file name has been specified in 'run_config', 'script' provided in ScriptRunConfig initialization will take precedence.\n"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "82d421c61bbe4e9486d80962be88267d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915?wsid=/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourcegroups/aml-quickstarts-170342/workspaces/quick-starts-ws-170342&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\", \"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"run_properties\": {\"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"created_utc\": \"2021-12-29T17:33:45.727386Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"bffb5af8-85db-4ae5-a247-ed8214f2b0a9\", \"user_agent\": \"python/3.6.9 (Linux-5.4.0-1063-azure-x86_64-with-debian-buster-sid) msrest/0.6.21 Hyperdrive.Service/1.0.0 Hyperdrive.SDK/core.1.36.0\", \"space_size\": \"18\", \"score\": \"0.9141122913505311\", \"best_child_run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1\", \"best_metric_status\": \"Succeeded\"}, \"tags\": {\"_aml_system_max_concurrent_jobs\": \"4\", \"_aml_system_max_total_jobs\": \"4\", \"_aml_system_max_duration_minutes\": \"10080\", \"_aml_system_policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 1, \\\"delay_evaluation\\\": 5, \\\"slack_factor\\\": 0.15}}\", \"_aml_system_generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--C\\\": [\\\"choice\\\", [[0.01, 0.1, 1]]], \\\"--max_iter\\\": [\\\"choice\\\", [[20, 40, 60, 100, 150, 200]]]}}\", \"_aml_system_primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"_aml_system_platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/udacity-project\\\", \\\"SubscriptionId\\\": \\\"3d1a56d2-7c81-4118-9790-f85d1acf0c77\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-170342\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-170342\\\", \\\"ExperimentName\\\": \\\"udacity-project\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [], \\\"target\\\": \\\"project1\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"priority\\\": null, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": true, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"sklearn:0.20.3-cpu\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": \\\"viennaprivate.azurecr.io\\\", \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"docker\\\": {\\\"useDocker\\\": false, \\\"sharedVolumes\\\": true, \\\"arguments\\\": [], \\\"shmSize\\\": \\\"2g\\\"}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"pytorch\\\": {\\\"communicationBackend\\\": \\\"nccl\\\", \\\"processCount\\\": null, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {}, \\\"datacaches\\\": [], \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": \\\"Standard_d2_v2\\\", \\\"vmPriority\\\": \\\"lowpriority\\\", \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}, \\\"kubernetescompute\\\": {\\\"instanceType\\\": null}, \\\"credentialPassthrough\\\": false, \\\"command\\\": \\\"\\\", \\\"environmentVariables\\\": {}, \\\"applicationEndpoints\\\": {}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"bffb5af8-85db-4ae5-a247-ed8214f2b0a9\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"c76680d6-2e30-46ca-96c2-f39884402ce2\\\", \\\"amlClientSessionId\\\": \\\"553c9424-287a-433c-8c8b-70b1739a9297\\\", \\\"subscriptionId\\\": \\\"3d1a56d2-7c81-4118-9790-f85d1acf0c77\\\", \\\"estimator\\\": \\\"SKLearn\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 4, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": \\\"Standard_d2_v2\\\"}}}\", \"_aml_system_resume_child_runs\": \"null\", \"_aml_system_all_jobs_generated\": \"true\", \"_aml_system_cancellation_requested\": \"false\", \"_aml_system_progress_metadata_evaluation_timestamp\": \"\\\"2021-12-29T17:33:47.150889\\\"\", \"_aml_system_progress_metadata_digest\": \"\\\"398de651cd6bbb5d764111a2007675560fb392ab2d81e643e34605a3818fdee6\\\"\", \"_aml_system_progress_metadata_active_timestamp\": \"\\\"2021-12-29T17:33:47.150889\\\"\", \"_aml_system_optimizer_state_artifact\": \"null\", \"_aml_system_outdated_optimizer_state_artifacts\": \"\\\"[]\\\"\", \"_aml_system_HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 60}\", \"_aml_system_HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 60}\", \"_aml_system_HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 200}\", \"_aml_system_final_best_metric_update_retry_count\": \"1\"}, \"end_time_utc\": \"2021-12-29T17:41:57.683136Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915/azureml-logs/hyperdrive.txt?sv=2019-07-07&sr=b&sig=PoEtKE15M3FCcG1QNRXGJSicAW42%2BIzK%2Bfv7s5wsIBU%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T18%3A14%3A52Z&se=2021-12-30T02%3A24%3A52Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:08:11\", \"run_number\": \"2\", \"run_queued_details\": {\"status\": \"Completed\", \"details\": null}, \"hyper_parameters\": {\"--C\": [\"choice\", [[0.01, 0.1, 1]]], \"--max_iter\": [\"choice\", [[20, 40, 60, 100, 150, 200]]]}}, \"child_runs\": [{\"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1\", \"run_number\": 3, \"metric\": 0.91411229, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-12-29T17:38:04.289879Z\", \"end_time\": \"2021-12-29T17:40:50.987749Z\", \"created_time\": \"2021-12-29T17:34:48.025618Z\", \"created_time_dt\": \"2021-12-29T17:34:48.025618Z\", \"duration\": \"0:06:02\", \"hyperdrive_id\": \"e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"arguments\": null, \"param_--C\": 1, \"param_--max_iter\": 60, \"best_metric\": 0.91411229}, {\"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0\", \"run_number\": 4, \"metric\": 0.91213961, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-12-29T17:38:09.100569Z\", \"end_time\": \"2021-12-29T17:41:00.332784Z\", \"created_time\": \"2021-12-29T17:34:48.057346Z\", \"created_time_dt\": \"2021-12-29T17:34:48.057346Z\", \"duration\": \"0:06:12\", \"hyperdrive_id\": \"e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"arguments\": null, \"param_--C\": 0.01, \"param_--max_iter\": 60, \"best_metric\": 0.91411229}, {\"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2\", \"run_number\": 5, \"metric\": 0.90804249, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-12-29T17:38:01.984345Z\", \"end_time\": \"2021-12-29T17:40:51.301704Z\", \"created_time\": \"2021-12-29T17:34:48.058707Z\", \"created_time_dt\": \"2021-12-29T17:34:48.058707Z\", \"duration\": \"0:06:03\", \"hyperdrive_id\": \"e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"arguments\": null, \"param_--C\": 0.01, \"param_--max_iter\": 100, \"best_metric\": 0.91411229}, {\"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3\", \"run_number\": 6, \"metric\": 0.90591806, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-12-29T17:38:03.800663Z\", \"end_time\": \"2021-12-29T17:40:54.87018Z\", \"created_time\": \"2021-12-29T17:34:48.133144Z\", \"created_time_dt\": \"2021-12-29T17:34:48.133144Z\", \"duration\": \"0:06:06\", \"hyperdrive_id\": \"e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"arguments\": null, \"param_--C\": 0.1, \"param_--max_iter\": 200, \"best_metric\": 0.91411229}], \"children_metrics\": {\"categories\": [0], \"series\": {\"Regularization Strength:\": [{\"categories\": [3, 4, 5, 6], \"mode\": \"markers\", \"name\": \"Regularization Strength:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [1.0, 0.01, 0.01, 0.1]}, {\"categories\": [3, 4, 5, 6], \"mode\": \"lines\", \"name\": \"Regularization Strength:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [1.0, 1.0, 1.0, 1.0]}], \"Max iterations:\": [{\"categories\": [3, 4, 5, 6], \"mode\": \"markers\", \"name\": \"Max iterations:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [60, 60, 100, 200]}, {\"categories\": [3, 4, 5, 6], \"mode\": \"lines\", \"name\": \"Max iterations:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [60, 60, 100, 200]}], \"Accuracy\": [{\"categories\": [3, 4, 5, 6], \"mode\": \"markers\", \"name\": \"Accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9141122913505311, 0.9121396054628225, 0.9080424886191198, 0.9059180576631259]}, {\"categories\": [3, 4, 5, 6], \"mode\": \"lines\", \"name\": \"Accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9141122913505311, 0.9141122913505311, 0.9141122913505311, 0.9141122913505311]}]}, \"metricName\": null, \"primaryMetricName\": \"Accuracy\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915\", \"categories\": [0], \"series\": [{\"data\": [{\"time_elapse\": [460, 460], \"metric_value\": [0.9141122913505311, 0.9141122913505311], \"metric_name\": [\"Accuracy\", \"Accuracy\"], \"run_id\": [\"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1\", \"HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1\"], \"final\": [false, true]}]}]}], \"run_logs\": \"[2021-12-29T17:33:46.108520][API][INFO]Experiment created\\r\\n[2021-12-29T17:33:47.420585][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2021-12-29T17:33:47.755027][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2021-12-29T17:34:17.444161][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2021-12-29T17:34:17.756523][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2021-12-29T17:34:47.4378353Z][SCHEDULER][INFO]Scheduling job, id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0'\\r\\n[2021-12-29T17:34:47.4600980Z][SCHEDULER][INFO]Scheduling job, id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3'\\r\\n[2021-12-29T17:34:47.484929][GENERATOR][INFO]Max number of jobs '4' reached for experiment.\\r\\n[2021-12-29T17:34:47.4636709Z][SCHEDULER][INFO]Scheduling job, id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2'\\r\\n[2021-12-29T17:34:47.4389777Z][SCHEDULER][INFO]Scheduling job, id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1'\\r\\n[2021-12-29T17:34:47.639393][GENERATOR][INFO]All jobs generated.\\r\\n[2021-12-29T17:34:48.1049654Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1'\\r\\n[2021-12-29T17:34:48.1320280Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0'\\r\\n[2021-12-29T17:34:48.2206647Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3'\\r\\n[2021-12-29T17:34:48.1987566Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2'\\r\\n[2021-12-29T17:38:16.514806][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:38:46.584049][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:39:16.606072][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:39:46.566360][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:40:16.598175][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:40:46.654934][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_0, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_2, https://southcentralus.experiments.azureml.net/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/experiments/**SCRUBBED**/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-12-29T17:41:57.869347][CONTROLLER][INFO]Experiment was 'ExperimentStatus.RUNNING', is 'ExperimentStatus.FINISHED'.\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.36.0\"}, \"loading\": false}"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "RunId: HD_e6329c9f-9c0d-4f05-ba62-227b182bc915\nWeb View: https://ml.azure.com/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915?wsid=/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourcegroups/aml-quickstarts-170342/workspaces/quick-starts-ws-170342&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\n\nStreaming azureml-logs/hyperdrive.txt\n=====================================\n\n\"<START>[2021-12-29T17:33:46.108520][API][INFO]Experiment created<END>\\n\"\"<START>[2021-12-29T17:33:47.420585][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space<END>\\n\"\"<START>[2021-12-29T17:33:47.755027][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.<END>\\n\"\n\nExecution Summary\n=================\nRunId: HD_e6329c9f-9c0d-4f05-ba62-227b182bc915\nWeb View: https://ml.azure.com/runs/HD_e6329c9f-9c0d-4f05-ba62-227b182bc915?wsid=/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourcegroups/aml-quickstarts-170342/workspaces/quick-starts-ws-170342&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\n\n"
        },
        {
          "output_type": "execute_result",
          "execution_count": 4,
          "data": {
            "text/plain": "{'runId': 'HD_e6329c9f-9c0d-4f05-ba62-227b182bc915',\n 'target': 'project1',\n 'status': 'Completed',\n 'startTimeUtc': '2021-12-29T17:33:45.809269Z',\n 'endTimeUtc': '2021-12-29T17:41:57.683136Z',\n 'services': {},\n 'properties': {'primary_metric_config': '{\"name\": \"Accuracy\", \"goal\": \"maximize\"}',\n  'resume_from': 'null',\n  'runTemplate': 'HyperDrive',\n  'azureml.runsource': 'hyperdrive',\n  'platform': 'AML',\n  'ContentSnapshotId': 'bffb5af8-85db-4ae5-a247-ed8214f2b0a9',\n  'user_agent': 'python/3.6.9 (Linux-5.4.0-1063-azure-x86_64-with-debian-buster-sid) msrest/0.6.21 Hyperdrive.Service/1.0.0 Hyperdrive.SDK/core.1.36.0',\n  'space_size': '18',\n  'score': '0.9141122913505311',\n  'best_child_run_id': 'HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1',\n  'best_metric_status': 'Succeeded'},\n 'inputDatasets': [],\n 'outputDatasets': [],\n 'logFiles': {'azureml-logs/hyperdrive.txt': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915/azureml-logs/hyperdrive.txt?sv=2019-07-07&sr=b&sig=M47CYEXLkGNtPFigvNbTmqCp7SF0fgsiyMKblY4tEoU%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A14Z&se=2021-12-30T01%3A42%3A14Z&sp=r'},\n 'submittedBy': 'ODL_User 170342'}"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Current provisioning state of AmlCompute is \"Deleting\"\n\n"
        }
      ],
      "execution_count": 4,
      "metadata": {
        "gather": {
          "logged": 1640799769924
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "# Get your best run and save the model from that run.\n",
        "\n",
        "hyperdrive_run_best = hyperdrive_run.get_best_run_by_primary_metric()\n",
        "\n",
        "print('best run details: \\n', hyperdrive_run_best.get_details())\n",
        "print('best run metrics: \\n', hyperdrive_run_best.get_metrics())\n",
        "\n",
        "hyperdrive_model = hyperdrive_run_best.register_model(\n",
        "    model_name='hyperdrive_run_best',\n",
        "    model_path='/azureml-logs/model.joblib')"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "best run details: \n {'runId': 'HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1', 'target': 'project1', 'status': 'Completed', 'startTimeUtc': '2021-12-29T17:38:04.289879Z', 'endTimeUtc': '2021-12-29T17:40:50.987749Z', 'services': {}, 'properties': {'_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'bffb5af8-85db-4ae5-a247-ed8214f2b0a9', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json'}, 'inputDatasets': [], 'outputDatasets': [], 'runDefinition': {'script': 'train.py', 'command': '', 'useAbsolutePath': False, 'arguments': ['--C', '1', '--max_iter', '60'], 'sourceDirectoryDataStore': None, 'framework': 'Python', 'communicator': 'None', 'target': 'project1', 'dataReferences': {}, 'data': {}, 'outputData': {}, 'datacaches': [], 'jobName': None, 'maxRunDurationSeconds': None, 'nodeCount': 1, 'instanceTypes': [], 'priority': None, 'credentialPassthrough': False, 'identity': None, 'environment': {'name': 'Experiment udacity-project Environment', 'version': 'Autosave_2021-12-29T17:34:47Z_7e2de2a1', 'python': {'interpreterPath': 'python', 'userManagedDependencies': True, 'condaDependencies': {'name': 'project_environment', 'dependencies': ['python=3.6.2', {'pip': ['azureml-defaults']}], 'channels': ['anaconda', 'conda-forge']}, 'baseCondaEnvironment': None}, 'environmentVariables': {'EXAMPLE_ENV_VAR': 'EXAMPLE_VALUE'}, 'docker': {'baseImage': 'sklearn:0.20.3-cpu', 'platform': {'os': 'Linux', 'architecture': 'amd64'}, 'baseDockerfile': None, 'baseImageRegistry': {'address': 'viennaprivate.azurecr.io', 'username': None, 'password': None}, 'enabled': True, 'arguments': []}, 'spark': {'repositories': [], 'packages': [], 'precachePackages': False}, 'inferencingStackVersion': None}, 'history': {'outputCollection': True, 'directoriesToWatch': ['logs'], 'enableMLflowTracking': True, 'snapshotProject': True}, 'spark': {'configuration': {'spark.app.name': 'Azure ML Experiment', 'spark.yarn.maxAppAttempts': '1'}}, 'parallelTask': {'maxRetriesPerWorker': 0, 'workerCountPerNode': 1, 'terminalExitCodes': None, 'configuration': {}}, 'amlCompute': {'name': None, 'vmSize': 'Standard_d2_v2', 'vmPriority': 'Lowpriority', 'retainCluster': False, 'clusterMaxNodeCount': 1}, 'aiSuperComputer': {'instanceType': 'AISupercomputer.D2', 'imageVersion': 'pytorch-1.7.0', 'location': None, 'aiSuperComputerStorageData': None, 'interactive': False, 'scalePolicy': None, 'virtualClusterArmId': None, 'tensorboardLogDirectory': None, 'sshPublicKey': None, 'sshPublicKeys': None, 'enableAzmlInt': True, 'priority': 'Medium', 'slaTier': 'Standard', 'userAlias': None}, 'kubernetesCompute': {'instanceType': None}, 'tensorflow': {'workerCount': 1, 'parameterServerCount': 1}, 'mpi': {'processCountPerNode': 1}, 'pyTorch': {'communicationBackend': 'nccl', 'processCount': None}, 'hdi': {'yarnDeployMode': 'Cluster'}, 'containerInstance': {'region': None, 'cpuCores': 2.0, 'memoryGb': 3.5}, 'exposedPorts': None, 'docker': {'useDocker': False, 'sharedVolumes': True, 'shmSize': '2g', 'arguments': []}, 'cmk8sCompute': {'configuration': {}}, 'commandReturnCodeConfig': {'returnCode': 'Zero', 'successfulReturnCodes': []}, 'environmentVariables': {}, 'applicationEndpoints': {}, 'parameters': [], 'dataBricks': {'workers': 0, 'minimumWorkerCount': 0, 'maxMumWorkerCount': 0, 'sparkVersion': '4.0.x-scala2.11', 'nodeTypeId': 'Standard_D3_v2', 'sparkConf': {}, 'sparkEnvVars': {}, 'instancePoolId': None, 'timeoutSeconds': 0, 'jarLibraries': [], 'eggLibraries': [], 'whlLibraries': [], 'pypiLibraries': [], 'rCranLibraries': [], 'mavenLibraries': [], 'linkedADBWorkspaceMetadata': None, 'databrickResourceId': None, 'autoScale': False}}, 'logFiles': {'azureml-logs/55_azureml-execution-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/55_azureml-execution-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt?sv=2019-07-07&sr=b&sig=xUQhoci54KSw0LOYf5gxvDuBqkVGXCtd7sU9GXuUyEw%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'azureml-logs/65_job_prep-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/65_job_prep-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt?sv=2019-07-07&sr=b&sig=tblb9IrSyzNlxOlVuzVxaHBcTMO1NeRF9mfOkgp33EU%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'azureml-logs/70_driver_log.txt': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/70_driver_log.txt?sv=2019-07-07&sr=b&sig=Zg7IyK3O3UrWnwE1itX6L6ZvWOoW8LwcqfbrTrFdvNw%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'azureml-logs/75_job_post-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/75_job_post-tvmps_3fc1064cd9f78bff2efcfd0e4ca8bb09d2ee1581ba326f06b07b4ffcd18f9879_d.txt?sv=2019-07-07&sr=b&sig=8Vx6DyjH3Cy0veizBkX02qnVQtubxnaLB%2FldaHEjH2M%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'azureml-logs/process_info.json': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/process_info.json?sv=2019-07-07&sr=b&sig=DnMaXvNjKw3hQt0B7xPI7HI1zuWMMfpgyo3XDooeMtE%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'azureml-logs/process_status.json': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/azureml-logs/process_status.json?sv=2019-07-07&sr=b&sig=CQUTsWRu6r%2FeNh3ObJDFrHmIDZDlUxmZmlJzZcniPrI%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'logs/azureml/112_azureml.log': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/logs/azureml/112_azureml.log?sv=2019-07-07&sr=b&sig=6eYNchXp6BhAYqmJQDJz0K%2BFolggYLPaTeybtK5ZUNs%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'logs/azureml/job_prep_azureml.log': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/logs/azureml/job_prep_azureml.log?sv=2019-07-07&sr=b&sig=9LJoKJ%2BZoY%2B9m0DcFKxQrxzZ%2BTG2JI0l7HyvKQ2iFfQ%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r', 'logs/azureml/job_release_azureml.log': 'https://mlstrg170342.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_e6329c9f-9c0d-4f05-ba62-227b182bc915_1/logs/azureml/job_release_azureml.log?sv=2019-07-07&sr=b&sig=LlVF4hKDZOU%2FR3ZzLbWEhSohoIUFFo6NZzX1%2BohpLKs%3D&skoid=23dce4b3-6435-453a-9793-1f31c57abdbf&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2021-12-29T17%3A23%3A47Z&ske=2021-12-31T01%3A33%3A47Z&sks=b&skv=2019-07-07&st=2021-12-29T17%3A32%3A50Z&se=2021-12-30T01%3A42%3A50Z&sp=r'}, 'submittedBy': 'ODL_User 170342'}\nbest run metrics: \n {'Regularization Strength:': 1.0, 'Max iterations:': 60, 'Accuracy': 0.9141122913505311}\n"
        }
      ],
      "execution_count": 5,
      "metadata": {
        "gather": {
          "logged": 1640799772522
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
        "\n",
        "# Create TabularDataset using TabularDatasetFactory\n",
        "# Data is available at: \n",
        "path = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
        "\n",
        "ds = TabularDatasetFactory.from_delimited_files([path])"
      ],
      "outputs": [],
      "execution_count": 6,
      "metadata": {
        "gather": {
          "logged": 1640799796962
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from azureml.core import Dataset\n",
        "from train import clean_data\n",
        "\n",
        "# Use the clean_data function to clean your data.\n",
        "x, y = clean_data(ds)\n",
        "\n",
        "data = pd.concat([x, y], axis=1)\n",
        "train_data, test_data = train_test_split(data, test_size=0.2)\n",
        "train_data.to_csv(\"training/training_df.csv\")\n",
        "\n",
        "ds_training = ws.get_default_datastore()\n",
        "ds_training.upload(src_dir=\"training\", target_path=\"training_data\")\n",
        "\n",
        "train_ds = Dataset.Tabular.from_delimited_files(path=[(ds_training, (\"training_data/training_df.csv\"))])"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "WARNING:azureml.data._dataset_deprecation:\"Datastore.upload\" is deprecated after version 1.0.69. Please use \"Dataset.File.upload_directory\" to upload your files             from a local directory and create FileDataset in single method call. See Dataset API change notice at https://aka.ms/dataset-deprecation.\n"
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Uploading an estimated of 2 files\nUploading training/train.py\nUploaded training/train.py, 1 files out of an estimated total of 2\nUploading training/training_df.csv\nUploaded training/training_df.csv, 2 files out of an estimated total of 2\nUploaded 2 files\n"
        }
      ],
      "execution_count": 7,
      "metadata": {
        "gather": {
          "logged": 1640799830530
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.train.automl import AutoMLConfig\n",
        "\n",
        "# Set parameters for AutoMLConfig\n",
        "# NOTE: DO NOT CHANGE THE experiment_timeout_minutes PARAMETER OR YOUR INSTANCE WILL TIME OUT.\n",
        "# If you wish to run the experiment longer, you will need to run this notebook in your own\n",
        "# Azure tenant, which will incur personal costs.\n",
        "automl_config = AutoMLConfig(\n",
        "    experiment_timeout_minutes=30,\n",
        "    task=\"classification\",\n",
        "    primary_metric=\"accuracy\",\n",
        "    training_data=train_ds,\n",
        "    label_column_name=\"y\",\n",
        "    n_cross_validations=5,\n",
        "    enable_early_stopping=True,\n",
        "    enable_onnx_compatible_models=True\n",
        ")"
      ],
      "outputs": [],
      "execution_count": 8,
      "metadata": {
        "gather": {
          "logged": 1640799830713
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Submit your automl run\n",
        "\n",
        "automl_run = exp.submit(config=automl_config, show_output=True)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "No run_configuration provided, running on local with default configuration\nRunning in the active local environment.\n"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "<table style=\"width:100%\"><tr><th>Experiment</th><th>Id</th><th>Type</th><th>Status</th><th>Details Page</th><th>Docs Page</th></tr><tr><td>udacity-project</td><td>AutoML_4a96a079-41f2-4e85-88c1-6fcf9793cf8c</td><td>automl</td><td>Preparing</td><td><a href=\"https://ml.azure.com/runs/AutoML_4a96a079-41f2-4e85-88c1-6fcf9793cf8c?wsid=/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourcegroups/aml-quickstarts-170342/workspaces/quick-starts-ws-170342&amp;tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\" target=\"_blank\" rel=\"noopener\">Link to Azure Machine Learning studio</a></td><td><a href=\"https://docs.microsoft.com/en-us/python/api/overview/azure/ml/intro?view=azure-ml-py\" target=\"_blank\" rel=\"noopener\">Link to Documentation</a></td></tr></table>"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Current status: DatasetEvaluation. Gathering dataset statistics.\nCurrent status: FeaturesGeneration. Generating features for the dataset.\nCurrent status: DatasetFeaturization. Beginning to fit featurizers and featurize the dataset.\nCurrent status: DatasetFeaturizationCompleted. Completed fit featurizers and featurizing the dataset.\nCurrent status: DatasetBalancing. Performing class balancing sweeping\nCurrent status: DatasetCrossValidationSplit. Generating individually featurized CV splits.\n\n****************************************************************************************************\nDATA GUARDRAILS: \n\nTYPE:         Class balancing detection\nSTATUS:       ALERTED\nDESCRIPTION:  To decrease model bias, please cancel the current run and fix balancing problem.\n              Learn more about imbalanced data: https://aka.ms/AutomatedMLImbalancedData\nDETAILS:      Imbalanced data can lead to a falsely perceived positive effect of a model's accuracy because the input data has bias towards one class.\n+---------------------------------+---------------------------------+--------------------------------------+\n|Size of the smallest class       |Name/Label of the smallest class |Number of samples in the training data|\n+=================================+=================================+======================================+\n|2963                             |1                                |26360                                 |\n+---------------------------------+---------------------------------+--------------------------------------+\n\n****************************************************************************************************\n\nTYPE:         Missing feature values imputation\nSTATUS:       PASSED\nDESCRIPTION:  No feature missing values were detected in the training data.\n              Learn more about missing value imputation: https://aka.ms/AutomatedMLFeaturization\n\n****************************************************************************************************\n\nTYPE:         High cardinality feature detection\nSTATUS:       PASSED\nDESCRIPTION:  Your inputs were analyzed, and no high cardinality features were detected.\n              Learn more about high cardinality feature handling: https://aka.ms/AutomatedMLFeaturization\n\n****************************************************************************************************\nCurrent status: ModelSelection. Beginning model selection.\n\n****************************************************************************************************\nITERATION: The iteration being evaluated.\nPIPELINE: A summary description of the pipeline being evaluated.\nDURATION: Time taken for the current iteration.\nMETRIC: The result of computing score on the fitted pipeline.\nBEST: The best observed score thus far.\n****************************************************************************************************\n\n ITERATION   PIPELINE                                       DURATION      METRIC      BEST\n         0   MaxAbsScaler LightGBM                          0:00:38       0.9147    0.9147\n         1   MaxAbsScaler XGBoostClassifier                 0:00:42       0.9140    0.9147\n         2   MaxAbsScaler ExtremeRandomTrees                0:00:37       0.7270    0.9147\n         3   SparseNormalizer XGBoostClassifier             0:00:35       0.9082    0.9147\n         4   MaxAbsScaler LightGBM                          0:00:31       0.9115    0.9147\n         5   MaxAbsScaler LightGBM                          0:00:29       0.8876    0.9147\n         6   StandardScalerWrapper XGBoostClassifier        0:00:35       0.9076    0.9147\n         7   MaxAbsScaler LogisticRegression                0:00:36       0.9077    0.9147\n         8   StandardScalerWrapper ExtremeRandomTrees       0:00:31       0.8877    0.9147\n         9   StandardScalerWrapper XGBoostClassifier        0:00:32       0.9066    0.9147\n        10   SparseNormalizer LightGBM                      0:00:29       0.9018    0.9147\n        11   StandardScalerWrapper XGBoostClassifier        0:00:32       0.9091    0.9147\n        12   MaxAbsScaler LogisticRegression                0:00:35       0.9080    0.9147\n        13   MaxAbsScaler SGD                               0:00:30       0.8521    0.9147\n        14   StandardScalerWrapper XGBoostClassifier        0:00:35       0.9134    0.9147\n        15   SparseNormalizer RandomForest                  0:00:51       0.7935    0.9147\n        16   StandardScalerWrapper LogisticRegression       0:00:31       0.9038    0.9147\n        17   StandardScalerWrapper RandomForest             0:00:38       0.9002    0.9147\n        18   StandardScalerWrapper XGBoostClassifier        0:00:37       0.9132    0.9147\n        19   TruncatedSVDWrapper RandomForest               0:03:07       0.8174    0.9147\n        20   TruncatedSVDWrapper RandomForest               0:04:05       0.8319    0.9147\n        21   StandardScalerWrapper XGBoostClassifier        0:00:36       0.9104    0.9147\n        22   VotingEnsemble                                 0:00:32       0.9160    0.9160\nStopping criteria reached at iteration 23. Ending experiment.\n****************************************************************************************************\nCurrent status: BestRunExplainModel. Best run model explanations started\nCurrent status: ModelExplanationDataSetSetup. Model explanations data setup completed\nCurrent status: PickSurrogateModel. Choosing LightGBM as the surrogate model for explanations\nCurrent status: EngineeredFeatureExplanations. Computation of engineered features started\nCurrent status: EngineeredFeatureExplanations. Computation of engineered features completed\nCurrent status: RawFeaturesExplanations. Computation of raw features started\nCurrent status: RawFeaturesExplanations. Computation of raw features completed\nCurrent status: BestRunExplainModel. Best run model explanations completed\n****************************************************************************************************\n"
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "INFO:interpret_community.common.explanation_utils:Using default datastore for uploads\n"
        }
      ],
      "execution_count": 9,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1640801926273
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.automl.runtime.onnx_convert import OnnxConverter\n",
        "# Retrieve and save your best automl model.\n",
        "\n",
        "print(\"autoML best run metrics: \\n \", automl_run.get_metrics())\n",
        "\n",
        "automl_best_run_onnx, automl_fitted_model_onnx = automl_run.get_output(return_onnx_model=True)\n",
        "OnnxConverter.save_onnx_model(automl_fitted_model_onnx, './outputs/best_automl_model.onnx')"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "autoML best run metrics: \n  {'AUC_macro': 0.9467679557033064, 'recall_score_macro': 0.7751928473527137, 'matthews_correlation': 0.5670620579230956, 'average_precision_score_macro': 0.8230771167409685, 'f1_score_weighted': 0.9146928964500969, 'f1_score_micro': 0.9160091047040971, 'weighted_accuracy': 0.9510994861744987, 'AUC_weighted': 0.9467679557033064, 'recall_score_micro': 0.9160091047040971, 'balanced_accuracy': 0.7751928473527137, 'f1_score_macro': 0.7829922108736164, 'average_precision_score_weighted': 0.9547391218703005, 'log_loss': 0.21743300025486412, 'precision_score_weighted': 0.9138036621356648, 'precision_score_micro': 0.9160091047040971, 'precision_score_macro': 0.7924669981355065, 'AUC_micro': 0.9802437678599801, 'accuracy': 0.9160091047040971, 'recall_score_weighted': 0.9160091047040971, 'average_precision_score_micro': 0.9809586359427449, 'norm_macro_recall': 0.5503856947054271}\n"
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "WARNING:urllib3.connectionpool:Retrying (Retry(total=2, connect=2, read=3, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f1349f92898>: Failed to establish a new connection: [Errno 111] Connection refused',)': /metric/v2.0/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/runs/AutoML_4a96a079-41f2-4e85-88c1-6fcf9793cf8c_2/full\nWARNING:urllib3.connectionpool:Retrying (Retry(total=2, connect=2, read=3, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f13a5cc6358>: Failed to establish a new connection: [Errno 111] Connection refused',)': /metric/v2.0/subscriptions/3d1a56d2-7c81-4118-9790-f85d1acf0c77/resourceGroups/aml-quickstarts-170342/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-170342/runs/AutoML_4a96a079-41f2-4e85-88c1-6fcf9793cf8c_5/full\n"
        }
      ],
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1640801996879
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# delete the cluster\r\n",
        "compute_target.delete()"
      ],
      "outputs": [],
      "execution_count": 11,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1640801997215
        }
      }
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python3"
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}