Skip to content

Commit

Permalink
Merge pull request #198 from mlrun/0.8.x-dev
Browse files Browse the repository at this point in the history 
0.8.x dev
  • Loading branch information
@aviaIguazio
aviaIguazio authored Nov 29, 2021
2 parents c991a77 + b23d231 commit fad845a
Show file tree
Hide file tree
Showing 6 changed files with 246 additions and 138 deletions.
56 changes: 24 additions & 32 deletions model-deployment-pipeline/functions/data-generator.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,26 +24,18 @@
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import subprocess\n",
"import pkg_resources\n",
"import IPython\n",
"\n",
"required = {'mlrun == 0.6.*', 'v3io == 0.6.*', 'faker'}\n",
"installed = {pkg.key for pkg in pkg_resources.working_set}\n",
"missing = required - installed\n",
"previously_installed = required.intersection(installed)\n",
"\n",
"if missing:\n",
" print(f'Installing {\",\".join(missing)}')\n",
" python = sys.executable\n",
" subprocess.check_call([python, '-m', 'pip', 'install', *missing], stdout=subprocess.DEVNULL)\n",
" print('Restarting kernel')\n",
" IPython.Application.instance().kernel.do_shutdown(True) #automatically restarts kernel\n",
"if previously_installed:\n",
" print(f'Already installed: {\",\".join(previously_installed)}')"
"!pip install faker"
]
},
{
"cell_type": "markdown",
"source": [
"##### Please make sure to restart your notebook's kernel before continuing"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -414,18 +406,18 @@
"> 2021-10-03 08:05:27,833 [info] Started building image: .mlrun/func-default-data-generator:latest\n",
"E1003 08:06:10.785107 1 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Deprecated.\n",
"\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors\n",
"\u001b[36mINFO\u001b[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0045] Built cross stage deps: map[] \n",
"\u001b[36mINFO\u001b[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0049] Executing 0 build triggers \n",
"\u001b[36mINFO\u001b[0m[0049] Unpacking rootfs as cmd RUN pip install faker requires it. \n",
"\u001b[36mINFO\u001b[0m[0072] RUN pip install faker \n",
"\u001b[36mINFO\u001b[0m[0072] Taking snapshot of full filesystem... \n",
"\u001b[36mINFO\u001b[0m[0079] cmd: /bin/sh \n",
"\u001b[36mINFO\u001b[0m[0079] args: [-c pip install faker] \n",
"\u001b[36mINFO\u001b[0m[0079] Running: [/bin/sh -c pip install faker] \n",
"\u001B[36mINFO\u001B[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0045] Built cross stage deps: map[] \n",
"\u001B[36mINFO\u001B[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0049] Executing 0 build triggers \n",
"\u001B[36mINFO\u001B[0m[0049] Unpacking rootfs as cmd RUN pip install faker requires it. \n",
"\u001B[36mINFO\u001B[0m[0072] RUN pip install faker \n",
"\u001B[36mINFO\u001B[0m[0072] Taking snapshot of full filesystem... \n",
"\u001B[36mINFO\u001B[0m[0079] cmd: /bin/sh \n",
"\u001B[36mINFO\u001B[0m[0079] args: [-c pip install faker] \n",
"\u001B[36mINFO\u001B[0m[0079] Running: [/bin/sh -c pip install faker] \n",
"Collecting faker\n",
" Downloading Faker-8.14.1-py3-none-any.whl (1.2 MB)\n",
"Requirement already satisfied: python-dateutil>=2.4 in /usr/local/lib/python3.7/site-packages (from faker) (2.8.2)\n",
Expand All @@ -436,7 +428,7 @@
"Successfully installed faker-8.14.1 text-unidecode-1.3\n",
"WARNING: You are using pip version 20.2.4; however, version 21.2.4 is available.\n",
"You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
"\u001b[36mINFO\u001b[0m[0083] Taking snapshot of full filesystem... \n"
"\u001B[36mINFO\u001B[0m[0083] Taking snapshot of full filesystem... \n"
]
},
{
Expand Down Expand Up @@ -505,4 +497,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
6 changes: 3 additions & 3 deletions model-deployment-pipeline/functions/event-handler.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
" setattr(context, 'batch_size', int(os.getenv('PARQUET_BATCH_SIZE', 1024)))\n",
"\n",
" setattr(context, 'timestamp_key', os.getenv('TS_KEY'))\n",
" setattr(context, 'timestamp_format', os.getenv('TS_FORMAT', '%Y-%m-%d %H:%M:%S.%f'))\n",
" setattr(context, 'timestamp_format', os.getenv('TS_FORMAT', '%Y-%m-%d %H:%M:%S.%f%z'))\n",
"\n",
" setattr(context, 'pq_partitions', ['pq_year', 'pq_month', 'pq_day', 'pq_hour'])\n",
"\n",
Expand Down Expand Up @@ -222,7 +222,7 @@
"\n",
"def flatten_inference_event(context, event):\n",
" # add parsed features to the event\n",
" feature_values = event['request']['instances'][0]\n",
" feature_values = event['request']['inputs'][0]\n",
" event.update(zip(context.features, feature_values))\n",
"\n",
" # add parsed predictions to the event\n",
Expand Down Expand Up @@ -541,4 +541,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
4 changes: 2 additions & 2 deletions model-deployment-pipeline/functions/stream-to-features.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@
" \n",
" feat_list = [resp.output.item.get(feat) for feat in features_list]\n",
" feat_list = [0 if v is None else v for v in feat_list]\n",
" return json.dumps({'user_id': user_id, 'instances': np.array(feat_list).reshape(1,-1).tolist()})\n",
" return json.dumps({'user_id': user_id, 'inputs': np.array(feat_list).reshape(1,-1).tolist()})\n",
"\n",
"\n",
"def write_to_output_stream(context, event):\n",
Expand Down Expand Up @@ -576,4 +576,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
88 changes: 44 additions & 44 deletions model-deployment-pipeline/project.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,18 +594,18 @@
"> 2021-10-03 09:10:52,044 [info] Started building image: .mlrun/func-model-deployment-pipeline-dani-data-generator:latest\n",
"E1003 09:11:34.660145 1 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Deprecated.\n",
"\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors\n",
"\u001b[36mINFO\u001b[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0045] Built cross stage deps: map[] \n",
"\u001b[36mINFO\u001b[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0049] Executing 0 build triggers \n",
"\u001b[36mINFO\u001b[0m[0049] Unpacking rootfs as cmd RUN pip install faker requires it. \n",
"\u001b[36mINFO\u001b[0m[0085] RUN pip install faker \n",
"\u001b[36mINFO\u001b[0m[0085] Taking snapshot of full filesystem... \n",
"\u001b[36mINFO\u001b[0m[0092] cmd: /bin/sh \n",
"\u001b[36mINFO\u001b[0m[0092] args: [-c pip install faker] \n",
"\u001b[36mINFO\u001b[0m[0092] Running: [/bin/sh -c pip install faker] \n",
"\u001B[36mINFO\u001B[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0045] Built cross stage deps: map[] \n",
"\u001B[36mINFO\u001B[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0049] Executing 0 build triggers \n",
"\u001B[36mINFO\u001B[0m[0049] Unpacking rootfs as cmd RUN pip install faker requires it. \n",
"\u001B[36mINFO\u001B[0m[0085] RUN pip install faker \n",
"\u001B[36mINFO\u001B[0m[0085] Taking snapshot of full filesystem... \n",
"\u001B[36mINFO\u001B[0m[0092] cmd: /bin/sh \n",
"\u001B[36mINFO\u001B[0m[0092] args: [-c pip install faker] \n",
"\u001B[36mINFO\u001B[0m[0092] Running: [/bin/sh -c pip install faker] \n",
"Collecting faker\n",
" Downloading Faker-8.14.1-py3-none-any.whl (1.2 MB)\n",
"Collecting text-unidecode==1.3\n",
Expand All @@ -616,7 +616,7 @@
"Successfully installed faker-8.14.1 text-unidecode-1.3\n",
"WARNING: You are using pip version 20.2.4; however, version 21.2.4 is available.\n",
"You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
"\u001b[36mINFO\u001b[0m[0098] Taking snapshot of full filesystem... \n"
"\u001B[36mINFO\u001B[0m[0098] Taking snapshot of full filesystem... \n"
]
},
{
Expand Down Expand Up @@ -1066,18 +1066,18 @@
"> 2021-10-03 09:14:49,375 [info] Started building image: .mlrun/func-model-deployment-pipeline-dani-get-data-snapshot:latest\n",
"E1003 09:15:32.433225 1 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Deprecated.\n",
"\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors\n",
"\u001b[36mINFO\u001b[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0045] Built cross stage deps: map[] \n",
"\u001b[36mINFO\u001b[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0049] Executing 0 build triggers \n",
"\u001b[36mINFO\u001b[0m[0049] Unpacking rootfs as cmd RUN pip install v3io-frames==0.8.* requires it. \n",
"\u001b[36mINFO\u001b[0m[0073] RUN pip install v3io-frames==0.8.* \n",
"\u001b[36mINFO\u001b[0m[0073] Taking snapshot of full filesystem... \n",
"\u001b[36mINFO\u001b[0m[0080] cmd: /bin/sh \n",
"\u001b[36mINFO\u001b[0m[0080] args: [-c pip install v3io-frames==0.8.*] \n",
"\u001b[36mINFO\u001b[0m[0080] Running: [/bin/sh -c pip install v3io-frames==0.8.*] \n",
"\u001B[36mINFO\u001B[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0042] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0045] Built cross stage deps: map[] \n",
"\u001B[36mINFO\u001B[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0049] Executing 0 build triggers \n",
"\u001B[36mINFO\u001B[0m[0049] Unpacking rootfs as cmd RUN pip install v3io-frames==0.8.* requires it. \n",
"\u001B[36mINFO\u001B[0m[0073] RUN pip install v3io-frames==0.8.* \n",
"\u001B[36mINFO\u001B[0m[0073] Taking snapshot of full filesystem... \n",
"\u001B[36mINFO\u001B[0m[0080] cmd: /bin/sh \n",
"\u001B[36mINFO\u001B[0m[0080] args: [-c pip install v3io-frames==0.8.*] \n",
"\u001B[36mINFO\u001B[0m[0080] Running: [/bin/sh -c pip install v3io-frames==0.8.*] \n",
"Requirement already satisfied: v3io-frames==0.8.* in /usr/local/lib/python3.7/site-packages (0.8.17)\n",
"Requirement already satisfied: googleapis-common-protos>=1.5.3 in /usr/local/lib/python3.7/site-packages (from v3io-frames==0.8.*) (1.53.0)\n",
"Requirement already satisfied: grpcio-tools<1.33,>=1.30 in /usr/local/lib/python3.7/site-packages (from v3io-frames==0.8.*) (1.32.0)\n",
Expand All @@ -1095,7 +1095,7 @@
"Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/site-packages (from pandas>=0.23.4->v3io-frames==0.8.*) (2.8.2)\n",
"WARNING: You are using pip version 20.2.4; however, version 21.2.4 is available.\n",
"You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
"\u001b[36mINFO\u001b[0m[0082] Taking snapshot of full filesystem... \n"
"\u001B[36mINFO\u001B[0m[0082] Taking snapshot of full filesystem... \n"
]
}
],
Expand Down Expand Up @@ -1638,18 +1638,18 @@
"> 2021-10-03 09:20:26,074 [info] Started building image: .mlrun/func-model-deployment-pipeline-dani-sklearn-classifier:latest\n",
"E1003 09:21:08.926917 1 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Deprecated.\n",
"\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors\n",
"\u001b[36mINFO\u001b[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0043] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0045] Built cross stage deps: map[] \n",
"\u001b[36mINFO\u001b[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001b[36mINFO\u001b[0m[0050] Executing 0 build triggers \n",
"\u001b[36mINFO\u001b[0m[0050] Unpacking rootfs as cmd RUN pip install scikit-learn==0.24.2 requires it. \n",
"\u001b[36mINFO\u001b[0m[0073] RUN pip install scikit-learn==0.24.2 \n",
"\u001b[36mINFO\u001b[0m[0073] Taking snapshot of full filesystem... \n",
"\u001b[36mINFO\u001b[0m[0080] cmd: /bin/sh \n",
"\u001b[36mINFO\u001b[0m[0080] args: [-c pip install scikit-learn==0.24.2] \n",
"\u001b[36mINFO\u001b[0m[0080] Running: [/bin/sh -c pip install scikit-learn==0.24.2] \n",
"\u001B[36mINFO\u001B[0m[0040] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0043] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0045] Built cross stage deps: map[] \n",
"\u001B[36mINFO\u001B[0m[0045] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0047] Retrieving image manifest mlrun/mlrun:0.7.0-rc7 \n",
"\u001B[36mINFO\u001B[0m[0050] Executing 0 build triggers \n",
"\u001B[36mINFO\u001B[0m[0050] Unpacking rootfs as cmd RUN pip install scikit-learn==0.24.2 requires it. \n",
"\u001B[36mINFO\u001B[0m[0073] RUN pip install scikit-learn==0.24.2 \n",
"\u001B[36mINFO\u001B[0m[0073] Taking snapshot of full filesystem... \n",
"\u001B[36mINFO\u001B[0m[0080] cmd: /bin/sh \n",
"\u001B[36mINFO\u001B[0m[0080] args: [-c pip install scikit-learn==0.24.2] \n",
"\u001B[36mINFO\u001B[0m[0080] Running: [/bin/sh -c pip install scikit-learn==0.24.2] \n",
"Collecting scikit-learn==0.24.2\n",
" Downloading scikit_learn-0.24.2-cp37-cp37m-manylinux2010_x86_64.whl (22.3 MB)\n",
"Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/site-packages (from scikit-learn==0.24.2) (1.19.5)\n",
Expand All @@ -1664,7 +1664,7 @@
"Successfully installed scikit-learn-0.24.2\n",
"WARNING: You are using pip version 20.2.4; however, version 21.2.4 is available.\n",
"You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
"\u001b[36mINFO\u001b[0m[0087] Taking snapshot of full filesystem... \n",
"\u001B[36mINFO\u001B[0m[0087] Taking snapshot of full filesystem... \n",
"> 2021-10-03 09:22:04,977 [info] starting run train uid=50c3165e4b2441138768ea223e3350f3 DB=http://mlrun-api:8080\n",
"> 2021-10-03 09:22:05,100 [info] Job is running in the background, pod: train-49fc5\n",
"> 2021-10-03 09:22:19,430 [info] best iteration=3, used criteria max.accuracy\n",
Expand Down Expand Up @@ -1911,7 +1911,6 @@
"if run_training:\n",
" project.set_function('hub://sklearn_classifier', 'train',image= 'mlrun/mlrun')\n",
" project.func('train').apply(mount_v3io())\n",
"# project.func('train').build_config(base_image='mlrun/mlrun', commands=['pip install scikit-learn==0.24.2'])\n",
" project.func('train').spec.build.commands = ['pip install scikit-learn==0.24.2']\n",
" # Configure the models to train\n",
" models = [\"sklearn.ensemble.RandomForestClassifier\",\n",
Expand Down Expand Up @@ -2307,8 +2306,9 @@
" serving.add_model('my_model', train_run.outputs.get('model'),class_name='ClassifierModel')\n",
"else:\n",
" serving.add_model('my_model', path.join(getcwd(), 'assets/model.pkl'),class_name='ClassifierModel')\n",
" \n",
"serving.set_envs({'INFERENCE_STREAM' : path.join(container, stream_configs['inference-stream']['path']) })\n",
"\n",
"serving.set_tracking(path.join('v3io:///',container, stream_configs['inference-stream']['path']),\n",
" stream_args={\"access_key\": getenv(\"V3IO_ACCESS_KEY\")})\n",
"\n",
"serving_stream = '/'.join(s.strip('/') for s in [web_api_users, stream_configs['serving-stream']['path']]) + '@ms'\n",
"serving.add_trigger('serving_stream',\n",
Expand Down Expand Up @@ -2369,7 +2369,7 @@
" 'PARQUET_TARGET_PATH' : inference_parquet_target_path,\n",
" 'PARQUET_BATCH_SIZE': 8192,\n",
" 'TS_KEY': 'when',\n",
" 'TS_FORMAT': '%Y-%m-%d %H:%M:%S.%f',\n",
" 'TS_FORMAT': '%Y-%m-%d %H:%M:%S.%f%z',\n",
" 'FEATURES': \",\".join(feature_list),\n",
" 'PREDICTIONS': 'about_to_churn',\n",
" 'CONTAINER': container}\n",
Expand Down Expand Up @@ -2431,4 +2431,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
Loading

0 comments on commit fad845a

Please sign in to comment.