howto/converting-to-mlrun/mlrun-code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Refactored As Operational Pipeline (with MLRun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: mlrun in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (0.8.0)\n",
      "Requirement already satisfied: lightgbm in /conda/lib/python3.7/site-packages (2.3.0)\n",
      "Requirement already satisfied: shapely in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (1.8.0)\n",
      "Requirement already satisfied: dask~=2.12 in /conda/lib/python3.7/site-packages (from mlrun) (2.30.0)\n",
      "Requirement already satisfied: google-auth<2.0dev,>=1.25.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.35.0)\n",
      "Requirement already satisfied: numpy<1.20.0,>=1.16.5 in /conda/lib/python3.7/site-packages (from mlrun) (1.19.2)\n",
      "Requirement already satisfied: fsspec~=2021.8.1 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (2021.8.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /conda/lib/python3.7/site-packages (from mlrun) (1.25.11)\n",
      "Requirement already satisfied: aiohttp~=3.6 in /conda/lib/python3.7/site-packages (from mlrun) (3.6.3)\n",
      "Requirement already satisfied: v3io-frames~=0.9.6 in /conda/lib/python3.7/site-packages (from mlrun) (0.9.6)\n",
      "Requirement already satisfied: alembic<1.6.0,~=1.4 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.5.8)\n",
      "Requirement already satisfied: GitPython~=3.0 in /conda/lib/python3.7/site-packages (from mlrun) (3.1.24)\n",
      "Requirement already satisfied: storey~=0.8.6; python_version >= \"3.7\" in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.8.8)\n",
      "Requirement already satisfied: tabulate<=0.8.3,>=0.8.0 in /conda/lib/python3.7/site-packages (from mlrun) (0.8.3)\n",
      "Requirement already satisfied: deepdiff~=5.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (5.6.0)\n",
      "Requirement already satisfied: ipykernel~=5.0 in /conda/lib/python3.7/site-packages (from mlrun) (5.3.4)\n",
      "Requirement already satisfied: orjson<3.4,>=3 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (3.3.1)\n",
      "Requirement already satisfied: fastapi~=0.67.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.67.0)\n",
      "Requirement already satisfied: v3io~=0.5.8 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.5.9)\n",
      "Requirement already satisfied: pyyaml~=5.1 in /conda/lib/python3.7/site-packages (from mlrun) (5.3.1)\n",
      "Requirement already satisfied: inflection~=0.5.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.5.1)\n",
      "Requirement already satisfied: kubernetes~=11.0 in /conda/lib/python3.7/site-packages (from mlrun) (11.0.0)\n",
      "Requirement already satisfied: cryptography<3.4,~=3.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (3.3.2)\n",
      "Requirement already satisfied: nest-asyncio~=1.0 in /conda/lib/python3.7/site-packages (from mlrun) (1.4.1)\n",
      "Requirement already satisfied: ipython<7.17,~=7.0 in /conda/lib/python3.7/site-packages (from mlrun) (7.16.1)\n",
      "Requirement already satisfied: kfp~=1.0.1 in /conda/lib/python3.7/site-packages (from mlrun) (1.0.1)\n",
      "Requirement already satisfied: semver~=2.13 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (2.13.0)\n",
      "Requirement already satisfied: v3iofs~=0.1.7 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.1.8)\n",
      "Requirement already satisfied: mergedeep~=1.3 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.3.4)\n",
      "Requirement already satisfied: chardet<4.0,>=3.0.2 in /conda/lib/python3.7/site-packages (from mlrun) (3.0.4)\n",
      "Requirement already satisfied: pyarrow~=1.0 in /conda/lib/python3.7/site-packages (from mlrun) (1.0.1)\n",
      "Requirement already satisfied: pydantic~=1.5 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.8.2)\n",
      "Requirement already satisfied: humanfriendly~=8.2 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (8.2)\n",
      "Requirement already satisfied: pandas~=1.2; python_version >= \"3.7\" in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.3.4)\n",
      "Requirement already satisfied: sqlalchemy~=1.3 in /conda/lib/python3.7/site-packages (from mlrun) (1.3.19)\n",
      "Requirement already satisfied: pymysql~=1.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (1.0.2)\n",
      "Requirement already satisfied: distributed<3,>=2.23 in /conda/lib/python3.7/site-packages (from mlrun) (2.30.0)\n",
      "Requirement already satisfied: nuclio-jupyter~=0.8.21 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from mlrun) (0.8.21)\n",
      "Requirement already satisfied: requests~=2.22 in /conda/lib/python3.7/site-packages (from mlrun) (2.24.0)\n",
      "Requirement already satisfied: click~=7.0 in /conda/lib/python3.7/site-packages (from mlrun) (7.1.2)\n",
      "Requirement already satisfied: scipy in /conda/lib/python3.7/site-packages (from lightgbm) (1.4.1)\n",
      "Requirement already satisfied: scikit-learn in /conda/lib/python3.7/site-packages (from lightgbm) (0.23.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun) (0.2.8)\n",
      "Requirement already satisfied: six>=1.9.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun) (1.15.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun) (4.1.1)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun) (41.0.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun) (4.6)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /conda/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun) (20.2.0)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in /conda/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun) (3.0.1)\n",
      "Requirement already satisfied: yarl<1.6.0,>=1.0 in /conda/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun) (1.5.1)\n",
      "Requirement already satisfied: multidict<5.0,>=4.5 in /conda/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun) (4.7.6)\n",
      "Requirement already satisfied: grpcio-tools<1.33,>=1.30 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.9.6->mlrun) (1.32.0)\n",
      "Requirement already satisfied: grpcio<1.33,>=1.30 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.9.6->mlrun) (1.32.0)\n",
      "Requirement already satisfied: googleapis-common-protos>=1.5.3 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.9.6->mlrun) (1.53.0)\n",
      "Requirement already satisfied: python-editor>=0.3 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from alembic<1.6.0,~=1.4->mlrun) (1.0.4)\n",
      "Requirement already satisfied: Mako in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from alembic<1.6.0,~=1.4->mlrun) (1.1.5)\n",
      "Requirement already satisfied: python-dateutil in /conda/lib/python3.7/site-packages (from alembic<1.6.0,~=1.4->mlrun) (2.8.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3; python_version < \"3.10\" in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from GitPython~=3.0->mlrun) (4.0.0)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /conda/lib/python3.7/site-packages (from GitPython~=3.0->mlrun) (4.0.9)\n",
      "Requirement already satisfied: ordered-set==4.0.2 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from deepdiff~=5.0->mlrun) (4.0.2)\n",
      "Requirement already satisfied: traitlets>=4.1.0 in /conda/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun) (4.3.3)\n",
      "Requirement already satisfied: jupyter-client in /conda/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun) (6.1.7)\n",
      "Requirement already satisfied: tornado>=4.2 in /conda/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun) (6.0.4)\n",
      "Requirement already satisfied: starlette==0.14.2 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from fastapi~=0.67.0->mlrun) (0.14.2)\n",
      "Requirement already satisfied: ujson>=3.0.0 in /conda/lib/python3.7/site-packages (from v3io~=0.5.8->mlrun) (4.2.0)\n",
      "Requirement already satisfied: future>=0.18.2 in /conda/lib/python3.7/site-packages (from v3io~=0.5.8->mlrun) (0.18.2)\n",
      "Requirement already satisfied: requests-oauthlib in /conda/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun) (1.3.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /conda/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun) (0.57.0)\n",
      "Requirement already satisfied: certifi>=14.05.14 in /conda/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun) (2021.10.8)\n",
      "Requirement already satisfied: cffi>=1.12 in /conda/lib/python3.7/site-packages (from cryptography<3.4,~=3.0->mlrun) (1.12.2)\n",
      "Requirement already satisfied: pygments in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (2.7.1)\n",
      "Requirement already satisfied: pickleshare in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (3.0.8)\n",
      "Requirement already satisfied: decorator in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (4.4.2)\n",
      "Requirement already satisfied: jedi>=0.10 in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (0.17.2)\n",
      "Requirement already satisfied: backcall in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (0.2.0)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /conda/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun) (4.8.0)\n",
      "Requirement already satisfied: strip-hints in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (0.1.8)\n",
      "Requirement already satisfied: cloudpickle in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (1.6.0)\n",
      "Requirement already satisfied: kfp-server-api<2.0.0,>=0.2.5 in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (1.5.0)\n",
      "Requirement already satisfied: jsonschema>=3.0.1 in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (3.2.0)\n",
      "Requirement already satisfied: google-cloud-storage>=1.13.0 in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (1.32.0)\n",
      "Requirement already satisfied: Deprecated in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (1.2.10)\n",
      "Requirement already satisfied: requests-toolbelt>=0.8.0 in /conda/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun) (0.9.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /conda/lib/python3.7/site-packages (from pandas~=1.2; python_version >= \"3.7\"->mlrun) (2020.1)\n",
      "Requirement already satisfied: zict>=0.1.3 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (2.0.0)\n",
      "Requirement already satisfied: psutil>=5.0 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (5.7.2)\n",
      "Requirement already satisfied: tblib>=1.6.0 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (1.7.0)\n",
      "Requirement already satisfied: toolz>=0.8.2 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (0.11.1)\n",
      "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (2.2.2)\n",
      "Requirement already satisfied: msgpack>=0.6.0 in /conda/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun) (1.0.0)\n",
      "Requirement already satisfied: nbconvert>=5.4 in /conda/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.21->mlrun) (6.0.7)\n",
      "Requirement already satisfied: notebook>=5.2.0 in /conda/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.21->mlrun) (6.1.6)\n",
      "Requirement already satisfied: boto3>=1.9 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.21->mlrun) (1.17.106)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /conda/lib/python3.7/site-packages (from requests~=2.22->mlrun) (2.8)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /conda/lib/python3.7/site-packages (from scikit-learn->lightgbm) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /conda/lib/python3.7/site-packages (from scikit-learn->lightgbm) (0.17.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.25.0->mlrun) (0.4.8)\n",
      "Requirement already satisfied: protobuf<4.0dev,>=3.5.0.post1 in /conda/lib/python3.7/site-packages (from grpcio-tools<1.33,>=1.30->v3io-frames~=0.9.6->mlrun) (3.19.0)\n",
      "Requirement already satisfied: MarkupSafe>=0.9.2 in /conda/lib/python3.7/site-packages (from Mako->alembic<1.6.0,~=1.4->mlrun) (1.1.1)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in /conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython~=3.0->mlrun) (5.0.0)\n",
      "Requirement already satisfied: ipython-genutils in /conda/lib/python3.7/site-packages (from traitlets>=4.1.0->ipykernel~=5.0->mlrun) (0.2.0)\n",
      "Requirement already satisfied: pyzmq>=13 in /conda/lib/python3.7/site-packages (from jupyter-client->ipykernel~=5.0->mlrun) (19.0.2)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /conda/lib/python3.7/site-packages (from jupyter-client->ipykernel~=5.0->mlrun) (4.6.3)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /conda/lib/python3.7/site-packages (from requests-oauthlib->kubernetes~=11.0->mlrun) (3.1.0)\n",
      "Requirement already satisfied: pycparser in /conda/lib/python3.7/site-packages (from cffi>=1.12->cryptography<3.4,~=3.0->mlrun) (2.19)\n",
      "Requirement already satisfied: wcwidth in /conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython<7.17,~=7.0->mlrun) (0.2.5)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /conda/lib/python3.7/site-packages (from jedi>=0.10->ipython<7.17,~=7.0->mlrun) (0.7.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /conda/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython<7.17,~=7.0->mlrun) (0.6.0)\n",
      "Requirement already satisfied: wheel in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from strip-hints->kfp~=1.0.1->mlrun) (0.37.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /conda/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp~=1.0.1->mlrun) (2.0.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /conda/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp~=1.0.1->mlrun) (0.17.3)\n",
      "Requirement already satisfied: google-cloud-core<2.0dev,>=1.4.1 in /conda/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun) (1.4.3)\n",
      "Requirement already satisfied: google-resumable-media<2.0dev,>=1.0.0 in /conda/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun) (1.1.0)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /conda/lib/python3.7/site-packages (from Deprecated->kfp~=1.0.1->mlrun) (1.12.1)\n",
      "Requirement already satisfied: heapdict in /conda/lib/python3.7/site-packages (from zict>=0.1.3->distributed<3,>=2.23->mlrun) (1.0.1)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (1.4.2)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.5.1)\n",
      "Requirement already satisfied: nbformat>=4.4 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (5.0.8)\n",
      "Requirement already satisfied: bleach in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (3.2.1)\n",
      "Requirement already satisfied: testpath in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.4.4)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.8.4)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.3)\n",
      "Requirement already satisfied: jupyterlab-pygments in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.1.2)\n",
      "Requirement already satisfied: jinja2>=2.4 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (2.11.2)\n",
      "Requirement already satisfied: defusedxml in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.6.0)\n",
      "Requirement already satisfied: argon2-cffi in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun) (20.1.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun) (0.9.1)\n",
      "Requirement already satisfied: Send2Trash in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun) (1.5.0)\n",
      "Requirement already satisfied: prometheus-client in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun) (0.8.0)\n",
      "Requirement already satisfied: botocore<1.21.0,>=1.20.106 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from boto3>=1.9->nuclio-jupyter~=0.8.21->mlrun) (1.20.106)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /conda/lib/python3.7/site-packages (from boto3>=1.9->nuclio-jupyter~=0.8.21->mlrun) (0.10.0)\n",
      "Requirement already satisfied: s3transfer<0.5.0,>=0.4.0 in /User/.pythonlibs/jupyter-avia/lib/python3.7/site-packages (from boto3>=1.9->nuclio-jupyter~=0.8.21->mlrun) (0.4.2)\n",
      "Requirement already satisfied: zipp>=0.5 in /conda/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema>=3.0.1->kfp~=1.0.1->mlrun) (3.3.1)\n",
      "Requirement already satisfied: google-api-core<2.0.0dev,>=1.19.0 in /conda/lib/python3.7/site-packages (from google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun) (1.22.2)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0; python_version >= \"3.5\" in /conda/lib/python3.7/site-packages (from google-resumable-media<2.0dev,>=1.0.0->google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun) (1.0.0)\n",
      "Requirement already satisfied: async-generator in /conda/lib/python3.7/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (1.10)\n",
      "Requirement already satisfied: packaging in /conda/lib/python3.7/site-packages (from bleach->nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (20.4)\n",
      "Requirement already satisfied: webencodings in /conda/lib/python3.7/site-packages (from bleach->nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (0.5.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /conda/lib/python3.7/site-packages (from packaging->bleach->nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun) (2.4.7)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# Install prerequisites\n",
    "%pip install mlrun lightgbm shapely"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an MLRun project and configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:26:48,429 [info] loaded project nyc-taxi-avia from MLRun DB\n",
      "Project name: nyc-taxi-avia\n",
      "Artifact path: v3io:///projects/{{run.project}}/artifacts\n"
     ]
    }
   ],
   "source": [
    "from os import path\n",
    "import mlrun\n",
    "\n",
    "project_name_base = 'nyc-taxi'\n",
    "\n",
    "project_name, artifact_path = mlrun.set_environment(project=project_name_base, user_project=True)\n",
    "\n",
    "print(f'Project name: {project_name}')\n",
    "print(f'Artifact path: {artifact_path}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Nuclio and MLRun Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: start-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgbm\n",
    "from mlrun.execution import MLClientCtx\n",
    "from mlrun.datastore import DataItem\n",
    "from pickle import dumps\n",
    "import shapely.wkt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zones_dict(zones_url):\n",
    "    zones_df = pd.read_csv(zones_url)\n",
    "    \n",
    "    # Remove unecessary fields\n",
    "    zones_df.drop(['Shape_Leng', 'Shape_Area', 'zone', 'LocationID', 'borough'], axis=1, inplace=True)\n",
    "    \n",
    "    # Convert DF to dictionary\n",
    "    zones_dict = zones_df.set_index('OBJECTID').to_dict('index')\n",
    "    \n",
    "    # Add lat/long to each zone\n",
    "    for zone in zones_dict:\n",
    "        shape = shapely.wkt.loads(zones_dict[zone]['the_geom'])\n",
    "        zones_dict[zone]['long'] = shape.centroid.x\n",
    "        zones_dict[zone]['lat'] = shape.centroid.y\n",
    "    \n",
    "    return zones_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_lat(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['lat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_long(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['long']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_df(df):\n",
    "    return df[(df.fare_amount > 0)  & (df.fare_amount <= 500) &\n",
    "             (df.PULocationID > 0) & (df.PULocationID <= 263) & \n",
    "             (df.DOLocationID > 0) & (df.DOLocationID <= 263)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Compute Haversine distance\n",
    "def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):\n",
    "    \"\"\"\n",
    "    Return distance along great radius between pickup and dropoff coordinates.\n",
    "    \"\"\"\n",
    "    #Define earth radius (km)\n",
    "    R_earth = 6371\n",
    "    #Convert degrees to radians\n",
    "    pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,\n",
    "                                                             [pickup_lat, pickup_lon, \n",
    "                                                              dropoff_lat, dropoff_lon])\n",
    "    #Compute distances along lat, lon dimensions\n",
    "    dlat = dropoff_lat - pickup_lat\n",
    "    dlon = dropoff_lon - pickup_lon\n",
    "    \n",
    "    #Compute haversine distance\n",
    "    a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat) * np.cos(dropoff_lat) * np.sin(dlon/2.0)**2\n",
    "    return 2 * R_earth * np.arcsin(np.sqrt(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def radian_conv(degree):\n",
    "    \"\"\"\n",
    "    Return radian.\n",
    "    \"\"\"\n",
    "    return  np.radians(degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_airport_dist(dataset):\n",
    "    \"\"\"\n",
    "    Return minumum distance from pickup or dropoff coordinates to each airport.\n",
    "    JFK: John F. Kennedy International Airport\n",
    "    EWR: Newark Liberty International Airport\n",
    "    LGA: LaGuardia Airport\n",
    "    SOL: Statue of Liberty \n",
    "    NYC: Newyork Central\n",
    "    \"\"\"\n",
    "    jfk_coord = (40.639722, -73.778889)\n",
    "    ewr_coord = (40.6925, -74.168611)\n",
    "    lga_coord = (40.77725, -73.872611)\n",
    "    sol_coord = (40.6892,-74.0445) # Statue of Liberty\n",
    "    nyc_coord = (40.7141667,-74.0063889) \n",
    "    \n",
    "    \n",
    "    pickup_lat = dataset['pickup_latitude']\n",
    "    dropoff_lat = dataset['dropoff_latitude']\n",
    "    pickup_lon = dataset['pickup_longitude']\n",
    "    dropoff_lon = dataset['dropoff_longitude']\n",
    "    \n",
    "    pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1]) \n",
    "    dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])\n",
    "    dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1]) \n",
    "    dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_sol = sphere_dist(pickup_lat, pickup_lon, sol_coord[0], sol_coord[1]) \n",
    "    dropoff_sol = sphere_dist(sol_coord[0], sol_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_nyc = sphere_dist(pickup_lat, pickup_lon, nyc_coord[0], nyc_coord[1]) \n",
    "    dropoff_nyc = sphere_dist(nyc_coord[0], nyc_coord[1], dropoff_lat, dropoff_lon)\n",
    "    \n",
    "    \n",
    "    \n",
    "    dataset['jfk_dist'] = pickup_jfk + dropoff_jfk\n",
    "    dataset['ewr_dist'] = pickup_ewr + dropoff_ewr\n",
    "    dataset['lga_dist'] = pickup_lga + dropoff_lga\n",
    "    dataset['sol_dist'] = pickup_sol + dropoff_sol\n",
    "    dataset['nyc_dist'] = pickup_nyc + dropoff_nyc\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datetime_info(dataset):\n",
    "    #Convert to datetime format\n",
    "    dataset['pickup_datetime'] = pd.to_datetime(dataset['tpep_pickup_datetime'],format=\"%Y-%m-%d %H:%M:%S\")\n",
    "    \n",
    "    dataset['hour'] = dataset.pickup_datetime.dt.hour\n",
    "    dataset['day'] = dataset.pickup_datetime.dt.day\n",
    "    dataset['month'] = dataset.pickup_datetime.dt.month\n",
    "    dataset['weekday'] = dataset.pickup_datetime.dt.weekday\n",
    "    dataset['year'] = dataset.pickup_datetime.dt.year\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_data(context : MLClientCtx, taxi_records_csv_path: DataItem, zones_csv_path: DataItem):\n",
    "    \n",
    "    context.logger.info('Reading taxi records data from {}'.format(taxi_records_csv_path))\n",
    "    taxi_records_dataset = taxi_records_csv_path.as_df()\n",
    "    \n",
    "    context.logger.info('Reading zones data from {}'.format(zones_csv_path))\n",
    "    zones_dataset = zones_csv_path.as_df()\n",
    "    \n",
    "    target_path = path.join(context.artifact_path, 'data')\n",
    "    context.logger.info('Saving datasets to {} ...'.format(target_path))\n",
    "\n",
    "    # Store the data sets in your artifacts database\n",
    "    context.log_dataset('nyc-taxi-dataset', df=taxi_records_dataset, format='csv',\n",
    "                        index=False, artifact_path=target_path)\n",
    "    context.log_dataset('zones-dataset', df=zones_dataset, format='csv',\n",
    "                        index=False, artifact_path=target_path)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zones_dict(zones_df):\n",
    "\n",
    "    # Remove unecessary fields\n",
    "    zones_df.drop(['Shape_Leng', 'Shape_Area', 'zone', 'LocationID', 'borough'], axis=1, inplace=True)\n",
    "    \n",
    "    # Convert DF to dictionary\n",
    "    zones_dict = zones_df.set_index('OBJECTID').to_dict('index')\n",
    "    \n",
    "    # Add lat/long to each zone\n",
    "    for zone in zones_dict:\n",
    "        shape = shapely.wkt.loads(zones_dict[zone]['the_geom'])\n",
    "        zones_dict[zone]['long'] = shape.centroid.x\n",
    "        zones_dict[zone]['lat'] = shape.centroid.y\n",
    "    \n",
    "    return zones_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_lat(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['lat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_long(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['long']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_dataset(context : MLClientCtx, taxi_records_csv_path: DataItem, zones_csv_path: DataItem):\n",
    "    \n",
    "    context.logger.info('Begin datasets transform')\n",
    "    \n",
    "    context.logger.info('zones_csv_path: ' + str(zones_csv_path))\n",
    "    \n",
    "    zones_df = zones_csv_path.as_df()    \n",
    "    \n",
    "    # Get zones dictionary\n",
    "    zones_dict = get_zones_dict(zones_df)\n",
    "    \n",
    "    train_df = taxi_records_csv_path.as_df()\n",
    "    \n",
    "    # Clean DF\n",
    "    train_df = clean_df(train_df)\n",
    "    \n",
    "    # Enrich DF\n",
    "    train_df['pickup_latitude'] = train_df.apply(lambda x: get_zone_lat(zones_dict, x['PULocationID']), axis=1 )\n",
    "    train_df['pickup_longitude'] = train_df.apply(lambda x: get_zone_long(zones_dict, x['PULocationID']), axis=1 )\n",
    "    train_df['dropoff_latitude'] = train_df.apply(lambda x: get_zone_lat(zones_dict, x['DOLocationID']), axis=1 )\n",
    "    train_df['dropoff_longitude'] = train_df.apply(lambda x: get_zone_long(zones_dict, x['DOLocationID']), axis=1 )\n",
    "\n",
    "    train_df = add_datetime_info(train_df)\n",
    "    train_df = add_airport_dist(train_df)\n",
    "\n",
    "    train_df['pickup_latitude'] = radian_conv(train_df['pickup_latitude'])\n",
    "    train_df['pickup_longitude'] = radian_conv(train_df['pickup_longitude'])\n",
    "    train_df['dropoff_latitude'] = radian_conv(train_df['dropoff_latitude'])\n",
    "    train_df['dropoff_longitude'] = radian_conv(train_df['dropoff_longitude'])\n",
    "\n",
    "    train_df.drop(['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime', 'congestion_surcharge', 'improvement_surcharge', 'pickup_datetime',\n",
    "                  'extra', 'mta_tax', 'tip_amount', 'tolls_amount', 'total_amount', 'RatecodeID', 'store_and_fwd_flag',\n",
    "                  'PULocationID', 'DOLocationID', 'payment_type'], \n",
    "                  axis=1, inplace=True, errors='ignore')\n",
    "    \n",
    "    # Save dataset to artifact\n",
    "    target_path = path.join(context.artifact_path, 'data')\n",
    "    context.log_dataset('nyc-taxi-dataset-transformed', df=train_df, artifact_path=target_path, format='csv')    \n",
    "    \n",
    "    context.logger.info('End dataset transform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'boosting_type':'gbdt',\n",
    "        'objective': 'regression',\n",
    "        'nthread': 4,\n",
    "        'num_leaves': 31,\n",
    "        'learning_rate': 0.05,\n",
    "        'max_depth': -1,\n",
    "        'subsample': 0.8,\n",
    "        'bagging_fraction' : 1,\n",
    "        'max_bin' : 5000 ,\n",
    "        'bagging_freq': 20,\n",
    "        'colsample_bytree': 0.6,\n",
    "        'metric': 'rmse',\n",
    "        'min_split_gain': 0.5,\n",
    "        'min_child_weight': 1,\n",
    "        'min_child_samples': 10,\n",
    "        'scale_pos_weight':1,\n",
    "        'zero_as_missing': True,\n",
    "        'seed':0,\n",
    "        'num_rounds':50000\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(context: MLClientCtx, input_ds: DataItem):\n",
    "    \n",
    "    context.logger.info('Begin training')\n",
    "    context.logger.info('LGBM version is ' + str(lgbm.__version__))\n",
    "    \n",
    "    train_df = input_ds.as_df()\n",
    "    \n",
    "    y = train_df['fare_amount']\n",
    "  \n",
    "    train_df = train_df.drop(columns=['fare_amount'])\n",
    "    train_df = train_df.drop(train_df.columns[[0]], axis=1)\n",
    "    x_train,x_test,y_train,y_test = train_test_split(train_df,y,random_state=123,test_size=0.10)\n",
    "    \n",
    "    train_set = lgbm.Dataset(x_train, y_train, silent=False,categorical_feature=['year','month','day','weekday'])\n",
    "    valid_set = lgbm.Dataset(x_test, y_test, silent=False,categorical_feature=['year','month','day','weekday'])\n",
    "    model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=500,verbose_eval=500, valid_sets=valid_set)\n",
    "    \n",
    "    context.log_model('FareModel',\n",
    "                     body=dumps(model),\n",
    "                     artifact_path=context.artifact_subpath(\"models\"),\n",
    "                     model_file=\"FareModel.pkl\")\n",
    "    \n",
    "    context.logger.info('End training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: end-code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Input Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import mlrun"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The example uses a CSVfile from MLRun demo data repository(by default stored in Wasabi object-store service)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxi_records_csv_path = mlrun.get_sample_path('data/Taxi/yellow_tripdata_2019-01_subset.csv')\n",
    "zones_csv_path = mlrun.get_sample_path('data/Taxi/taxi_zones.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Code to a Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxi_func = mlrun.code_to_function(name='taxi',\n",
    "                                   kind='job',\n",
    "                                   image='mlrun/mlrun',\n",
    "                                   requirements=['lightgbm', 'shapely'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run `fetch_data` Locally\n",
    "\n",
    "We can test out code locally, by calling the function with `local` parameter set to `True`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:26:58,684 [info] starting run taxi-fetch_data uid=b2702879c6d341ef9d287d304e19a3f9 DB=http://mlrun-api:8080\n",
      "> 2021-11-16 13:26:58,894 [info] Reading taxi records data from https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\n",
      "> 2021-11-16 13:27:04,132 [info] Reading zones data from https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\n",
      "> 2021-11-16 13:27:04,648 [info] Saving datasets to v3io:///projects/nyc-taxi-avia/artifacts/data ...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-avia</td>\n",
       "      <td><div title=\"b2702879c6d341ef9d287d304e19a3f9\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/b2702879c6d341ef9d287d304e19a3f9/overview\" target=\"_blank\" >...4e19a3f9</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Nov 16 13:26:58</td>\n",
       "      <td>completed</td>\n",
       "      <td>taxi-fetch_data</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=avia</div><div class=\"dictlist\">kind=</div><div class=\"dictlist\">owner=avia</div><div class=\"dictlist\">host=jupyter-avia-8774f9fc7-rbh9v</div></td>\n",
       "      <td><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\">taxi_records_csv_path</div><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result8655466a\" title=\"files/v3io/projects/nyc-taxi-avia/artifacts/data/nyc-taxi-dataset.csv\">nyc-taxi-dataset</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result8655466a\" title=\"files/v3io/projects/nyc-taxi-avia/artifacts/data/zones-dataset.csv\">zones-dataset</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result8655466a-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result8655466a-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result8655466a\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result8655466a-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/b2702879c6d341ef9d287d304e19a3f9/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:27:12,349 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "fetch_data_run = taxi_func.run(handler='fetch_data',\n",
    "                               inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                                       'zones_csv_path': zones_csv_path},\n",
    "                               local=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset': 'store://artifacts/nyc-taxi-avia/taxi-fetch_data_nyc-taxi-dataset:b2702879c6d341ef9d287d304e19a3f9',\n",
       " 'zones-dataset': 'store://artifacts/nyc-taxi-avia/taxi-fetch_data_zones-dataset:b2702879c6d341ef9d287d304e19a3f9'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_data_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run on the Cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Cluster Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an MLRun function and create a custom image for it (that uses shapely)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:27:12,558 [info] Started building image: .mlrun/func-nyc-taxi-avia-taxi:latest\n",
      "\u001b[36mINFO\u001b[0m[0000] Retrieving image manifest mlrun/mlrun:0.8.0  \n",
      "\u001b[36mINFO\u001b[0m[0000] Retrieving image mlrun/mlrun:0.8.0 from registry index.docker.io \n",
      "\u001b[36mINFO\u001b[0m[0000] Built cross stage deps: map[]                \n",
      "\u001b[36mINFO\u001b[0m[0000] Retrieving image manifest mlrun/mlrun:0.8.0  \n",
      "\u001b[36mINFO\u001b[0m[0000] Returning cached image manifest              \n",
      "\u001b[36mINFO\u001b[0m[0000] Executing 0 build triggers                   \n",
      "\u001b[36mINFO\u001b[0m[0000] Unpacking rootfs as cmd RUN python -m pip install lightgbm shapely requires it. \n",
      "\u001b[36mINFO\u001b[0m[0015] RUN python -m pip install lightgbm shapely   \n",
      "\u001b[36mINFO\u001b[0m[0015] Taking snapshot of full filesystem...        \n",
      "\u001b[36mINFO\u001b[0m[0026] cmd: /bin/sh                                 \n",
      "\u001b[36mINFO\u001b[0m[0026] args: [-c python -m pip install lightgbm shapely] \n",
      "\u001b[36mINFO\u001b[0m[0026] Running: [/bin/sh -c python -m pip install lightgbm shapely] \n",
      "Collecting lightgbm\n",
      "  Downloading lightgbm-3.3.1-py3-none-manylinux1_x86_64.whl (2.0 MB)\n",
      "Collecting shapely\n",
      "  Downloading Shapely-1.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.1 MB)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/site-packages (from lightgbm) (1.19.5)\n",
      "Requirement already satisfied: wheel in /usr/local/lib/python3.7/site-packages (from lightgbm) (0.36.2)\n",
      "Requirement already satisfied: scikit-learn!=0.22.0 in /usr/local/lib/python3.7/site-packages (from lightgbm) (0.23.2)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.7/site-packages (from lightgbm) (1.7.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/site-packages (from scikit-learn!=0.22.0->lightgbm) (1.1.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/site-packages (from scikit-learn!=0.22.0->lightgbm) (3.0.0)\n",
      "Installing collected packages: lightgbm, shapely\n",
      "Successfully installed lightgbm-3.3.1 shapely-1.8.0\n",
      "WARNING: You are using pip version 20.2.4; however, version 21.3.1 is available.\n",
      "You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
      "\u001b[36mINFO\u001b[0m[0029] Taking snapshot of full filesystem...        \n",
      "\u001b[36mINFO\u001b[0m[0030] RUN python -m pip install \"mlrun[complete]==0.8.0\" \n",
      "\u001b[36mINFO\u001b[0m[0030] cmd: /bin/sh                                 \n",
      "\u001b[36mINFO\u001b[0m[0030] args: [-c python -m pip install \"mlrun[complete]==0.8.0\"] \n",
      "\u001b[36mINFO\u001b[0m[0030] Running: [/bin/sh -c python -m pip install \"mlrun[complete]==0.8.0\"] \n",
      "Requirement already satisfied: mlrun[complete]==0.8.0 in /usr/local/lib/python3.7/site-packages (0.8.0)\n",
      "Requirement already satisfied: kfp~=1.0.1 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.0.4)\n",
      "Requirement already satisfied: distributed<3,>=2.23 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2.30.1)\n",
      "Requirement already satisfied: pymysql~=1.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.0.2)\n",
      "Requirement already satisfied: tabulate<=0.8.3,>=0.8.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.8.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.26.7)\n",
      "Requirement already satisfied: v3io~=0.5.8 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.5.9)\n",
      "Requirement already satisfied: cryptography<3.4,~=3.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (3.3.2)\n",
      "Requirement already satisfied: GitPython~=3.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (3.1.24)\n",
      "Requirement already satisfied: fsspec~=2021.8.1 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2021.8.1)\n",
      "Requirement already satisfied: dask~=2.12 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2.30.0)\n",
      "Requirement already satisfied: deepdiff~=5.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (5.6.0)\n",
      "Requirement already satisfied: requests~=2.22 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2.26.0)\n",
      "Requirement already satisfied: inflection~=0.5.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.5.1)\n",
      "Requirement already satisfied: kubernetes~=11.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (11.0.0)\n",
      "Requirement already satisfied: chardet<4.0,>=3.0.2 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (3.0.4)\n",
      "Requirement already satisfied: mergedeep~=1.3 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.3.4)\n",
      "Requirement already satisfied: ipython<7.17,~=7.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (7.16.1)\n",
      "Requirement already satisfied: orjson<3.4,>=3 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (3.3.1)\n",
      "Requirement already satisfied: pydantic~=1.5 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.8.2)\n",
      "Requirement already satisfied: aiohttp~=3.6 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (3.7.4.post0)\n",
      "Requirement already satisfied: alembic<1.6.0,~=1.4 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.5.8)\n",
      "Requirement already satisfied: pyarrow~=1.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.0.1)\n",
      "Requirement already satisfied: nest-asyncio~=1.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.5.1)\n",
      "Requirement already satisfied: sqlalchemy~=1.3 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.4.26)\n",
      "Requirement already satisfied: pyyaml~=5.1 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (5.4.1)\n",
      "Requirement already satisfied: click~=7.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (7.1.2)\n",
      "Requirement already satisfied: numpy<1.20.0,>=1.16.5 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.19.5)\n",
      "Requirement already satisfied: ipykernel~=5.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (5.5.6)\n",
      "Requirement already satisfied: storey~=0.8.6; python_version >= \"3.7\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.8.6)\n",
      "Requirement already satisfied: v3io-frames~=0.9.6 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.9.6)\n",
      "Requirement already satisfied: pandas~=1.2; python_version >= \"3.7\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.3.4)\n",
      "Requirement already satisfied: v3iofs~=0.1.7 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.1.7)\n",
      "Requirement already satisfied: fastapi~=0.67.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.67.0)\n",
      "Requirement already satisfied: humanfriendly~=8.2 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (8.2)\n",
      "Requirement already satisfied: google-auth<2.0dev,>=1.25.0 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.35.0)\n",
      "Requirement already satisfied: semver~=2.13 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2.13.0)\n",
      "Requirement already satisfied: nuclio-jupyter~=0.8.21 in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (0.8.21)\n",
      "Requirement already satisfied: azure-keyvault-secrets~=4.2; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (4.3.0)\n",
      "Requirement already satisfied: azure-identity~=1.5; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.7.0)\n",
      "Requirement already satisfied: azure-storage-blob~=12.0; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (12.9.0)\n",
      "Requirement already satisfied: bokeh~=2.3.0; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2.3.3)\n",
      "Requirement already satisfied: boto3<1.17.107,~=1.9; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.17.106)\n",
      "Requirement already satisfied: s3fs~=2021.8.1; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2021.8.1)\n",
      "Requirement already satisfied: adlfs~=2021.8.1; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2021.8.2)\n",
      "Requirement already satisfied: gcsfs~=2021.8.1; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (2021.8.1)\n",
      "Requirement already satisfied: aiobotocore~=1.4.0; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.4.2)\n",
      "Requirement already satisfied: botocore<1.20.107,>=1.20.106; extra == \"complete\" in /usr/local/lib/python3.7/site-packages (from mlrun[complete]==0.8.0) (1.20.106)\n",
      "Requirement already satisfied: strip-hints in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (0.1.10)\n",
      "Requirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (4.1.2)\n",
      "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (2.0.0)\n",
      "Requirement already satisfied: requests-toolbelt>=0.8.0 in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (0.9.1)\n",
      "Requirement already satisfied: google-cloud-storage>=1.13.0 in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (1.42.3)\n",
      "Requirement already satisfied: kfp-server-api<2.0.0,>=0.2.5 in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (1.7.0)\n",
      "Requirement already satisfied: Deprecated in /usr/local/lib/python3.7/site-packages (from kfp~=1.0.1->mlrun[complete]==0.8.0) (1.2.13)\n",
      "Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (1.7.0)\n",
      "Requirement already satisfied: zict>=0.1.3 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (2.0.0)\n",
      "Requirement already satisfied: msgpack>=0.6.0 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (1.0.2)\n",
      "Requirement already satisfied: tornado>=5; python_version < \"3.8\" in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (6.1)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (53.0.0)\n",
      "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (2.4.0)\n",
      "Requirement already satisfied: toolz>=0.8.2 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (0.11.1)\n",
      "Requirement already satisfied: psutil>=5.0 in /usr/local/lib/python3.7/site-packages (from distributed<3,>=2.23->mlrun[complete]==0.8.0) (5.8.0)\n",
      "Requirement already satisfied: future>=0.18.2 in /usr/local/lib/python3.7/site-packages (from v3io~=0.5.8->mlrun[complete]==0.8.0) (0.18.2)\n",
      "Requirement already satisfied: ujson>=3.0.0 in /usr/local/lib/python3.7/site-packages (from v3io~=0.5.8->mlrun[complete]==0.8.0) (4.2.0)\n",
      "Requirement already satisfied: six>=1.4.1 in /usr/local/lib/python3.7/site-packages (from cryptography<3.4,~=3.0->mlrun[complete]==0.8.0) (1.16.0)\n",
      "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.7/site-packages (from cryptography<3.4,~=3.0->mlrun[complete]==0.8.0) (1.15.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3; python_version < \"3.10\" in /usr/local/lib/python3.7/site-packages (from GitPython~=3.0->mlrun[complete]==0.8.0) (3.10.0.2)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/site-packages (from GitPython~=3.0->mlrun[complete]==0.8.0) (4.0.9)\n",
      "Requirement already satisfied: ordered-set==4.0.2 in /usr/local/lib/python3.7/site-packages (from deepdiff~=5.0->mlrun[complete]==0.8.0) (4.0.2)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0; python_version >= \"3\" in /usr/local/lib/python3.7/site-packages (from requests~=2.22->mlrun[complete]==0.8.0) (2.0.7)\n",
      "Requirement already satisfied: idna<4,>=2.5; python_version >= \"3\" in /usr/local/lib/python3.7/site-packages (from requests~=2.22->mlrun[complete]==0.8.0) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests~=2.22->mlrun[complete]==0.8.0) (2021.10.8)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun[complete]==0.8.0) (1.2.1)\n",
      "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun[complete]==0.8.0) (1.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.7/site-packages (from kubernetes~=11.0->mlrun[complete]==0.8.0) (2.8.2)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (3.0.21)\n",
      "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (5.1.1)\n",
      "Requirement already satisfied: pygments in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (2.10.0)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (5.1.0)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.7.5)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (4.8.0)\n",
      "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.7/site-packages (from ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.18.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun[complete]==0.8.0) (5.2.0)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun[complete]==0.8.0) (3.0.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun[complete]==0.8.0) (21.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/site-packages (from aiohttp~=3.6->mlrun[complete]==0.8.0) (1.7.0)\n",
      "Requirement already satisfied: Mako in /usr/local/lib/python3.7/site-packages (from alembic<1.6.0,~=1.4->mlrun[complete]==0.8.0) (1.1.5)\n",
      "Requirement already satisfied: python-editor>=0.3 in /usr/local/lib/python3.7/site-packages (from alembic<1.6.0,~=1.4->mlrun[complete]==0.8.0) (1.0.4)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/site-packages (from sqlalchemy~=1.3->mlrun[complete]==0.8.0) (4.8.1)\n",
      "Requirement already satisfied: greenlet!=0.4.17; python_version >= \"3\" and (platform_machine == \"aarch64\" or (platform_machine == \"ppc64le\" or (platform_machine == \"x86_64\" or (platform_machine == \"amd64\" or (platform_machine == \"AMD64\" or (platform_machine == \"win32\" or platform_machine == \"WIN32\")))))) in /usr/local/lib/python3.7/site-packages (from sqlalchemy~=1.3->mlrun[complete]==0.8.0) (1.1.2)\n",
      "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun[complete]==0.8.0) (0.2.0)\n",
      "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun[complete]==0.8.0) (7.0.6)\n",
      "Requirement already satisfied: grpcio<1.33,>=1.30 in /usr/local/lib/python3.7/site-packages (from storey~=0.8.6; python_version >= \"3.7\"->mlrun[complete]==0.8.0) (1.32.0)\n",
      "Requirement already satisfied: grpcio-tools<1.33,>=1.30 in /usr/local/lib/python3.7/site-packages (from storey~=0.8.6; python_version >= \"3.7\"->mlrun[complete]==0.8.0) (1.32.0)\n",
      "Requirement already satisfied: googleapis-common-protos>=1.5.3 in /usr/local/lib/python3.7/site-packages (from v3io-frames~=0.9.6->mlrun[complete]==0.8.0) (1.53.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/site-packages (from pandas~=1.2; python_version >= \"3.7\"->mlrun[complete]==0.8.0) (2021.3)\n",
      "Requirement already satisfied: starlette==0.14.2 in /usr/local/lib/python3.7/site-packages (from fastapi~=0.67.0->mlrun[complete]==0.8.0) (0.14.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.8.0) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.8.0) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.8.0) (4.2.4)\n",
      "Requirement already satisfied: notebook>=5.2.0 in /usr/local/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (6.4.5)\n",
      "Requirement already satisfied: nbconvert>=5.4 in /usr/local/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (6.2.0)\n",
      "Requirement already satisfied: azure-common~=1.1 in /usr/local/lib/python3.7/site-packages (from azure-keyvault-secrets~=4.2; extra == \"complete\"->mlrun[complete]==0.8.0) (1.1.27)\n",
      "Requirement already satisfied: azure-core<2.0.0,>=1.7.0 in /usr/local/lib/python3.7/site-packages (from azure-keyvault-secrets~=4.2; extra == \"complete\"->mlrun[complete]==0.8.0) (1.19.0)\n",
      "Requirement already satisfied: msrest>=0.6.21 in /usr/local/lib/python3.7/site-packages (from azure-keyvault-secrets~=4.2; extra == \"complete\"->mlrun[complete]==0.8.0) (0.6.21)\n",
      "Requirement already satisfied: msal<2.0.0,>=1.12.0 in /usr/local/lib/python3.7/site-packages (from azure-identity~=1.5; extra == \"complete\"->mlrun[complete]==0.8.0) (1.16.0)\n",
      "Requirement already satisfied: msal-extensions~=0.3.0 in /usr/local/lib/python3.7/site-packages (from azure-identity~=1.5; extra == \"complete\"->mlrun[complete]==0.8.0) (0.3.0)\n",
      "Requirement already satisfied: packaging>=16.8 in /usr/local/lib/python3.7/site-packages (from bokeh~=2.3.0; extra == \"complete\"->mlrun[complete]==0.8.0) (21.2)\n",
      "Requirement already satisfied: pillow>=7.1.0 in /usr/local/lib/python3.7/site-packages (from bokeh~=2.3.0; extra == \"complete\"->mlrun[complete]==0.8.0) (8.4.0)\n",
      "Requirement already satisfied: Jinja2>=2.9 in /usr/local/lib/python3.7/site-packages (from bokeh~=2.3.0; extra == \"complete\"->mlrun[complete]==0.8.0) (3.0.2)\n",
      "Requirement already satisfied: s3transfer<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/site-packages (from boto3<1.17.107,~=1.9; extra == \"complete\"->mlrun[complete]==0.8.0) (0.4.2)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.7/site-packages (from boto3<1.17.107,~=1.9; extra == \"complete\"->mlrun[complete]==0.8.0) (0.10.0)\n",
      "Requirement already satisfied: azure-datalake-store<0.1,>=0.0.46 in /usr/local/lib/python3.7/site-packages (from adlfs~=2021.8.1; extra == \"complete\"->mlrun[complete]==0.8.0) (0.0.52)\n",
      "Requirement already satisfied: google-auth-oauthlib in /usr/local/lib/python3.7/site-packages (from gcsfs~=2021.8.1; extra == \"complete\"->mlrun[complete]==0.8.0) (0.4.6)\n",
      "Requirement already satisfied: aioitertools>=0.5.1 in /usr/local/lib/python3.7/site-packages (from aiobotocore~=1.4.0; extra == \"complete\"->mlrun[complete]==0.8.0) (0.8.0)\n",
      "Requirement already satisfied: wrapt>=1.10.10 in /usr/local/lib/python3.7/site-packages (from aiobotocore~=1.4.0; extra == \"complete\"->mlrun[complete]==0.8.0) (1.13.3)\n",
      "Requirement already satisfied: wheel in /usr/local/lib/python3.7/site-packages (from strip-hints->kfp~=1.0.1->mlrun[complete]==0.8.0) (0.36.2)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp~=1.0.1->mlrun[complete]==0.8.0) (0.18.0)\n",
      "Requirement already satisfied: protobuf; python_version >= \"3.6\" in /usr/local/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun[complete]==0.8.0) (3.19.1)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0; python_version >= \"3.6\" in /usr/local/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun[complete]==0.8.0) (2.1.0)\n",
      "Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0; python_version >= \"3.6\" in /usr/local/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun[complete]==0.8.0) (2.1.0)\n",
      "Requirement already satisfied: google-api-core<3.0dev,>=1.29.0; python_version >= \"3.6\" in /usr/local/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun[complete]==0.8.0) (2.2.1)\n",
      "Requirement already satisfied: heapdict in /usr/local/lib/python3.7/site-packages (from zict>=0.1.3->distributed<3,>=2.23->mlrun[complete]==0.8.0) (1.0.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/site-packages (from cffi>=1.12->cryptography<3.4,~=3.0->mlrun[complete]==0.8.0) (2.20)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython~=3.0->mlrun[complete]==0.8.0) (5.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/site-packages (from requests-oauthlib->kubernetes~=11.0->mlrun[complete]==0.8.0) (3.1.1)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.2.5)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.7.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/site-packages (from jedi>=0.10->ipython<7.17,~=7.0->mlrun[complete]==0.8.0) (0.8.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.7/site-packages (from Mako->alembic<1.6.0,~=1.4->mlrun[complete]==0.8.0) (2.0.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->sqlalchemy~=1.3->mlrun[complete]==0.8.0) (3.6.0)\n",
      "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/site-packages (from jupyter-client->ipykernel~=5.0->mlrun[complete]==0.8.0) (22.3.0)\n",
      "Requirement already satisfied: entrypoints in /usr/local/lib/python3.7/site-packages (from jupyter-client->ipykernel~=5.0->mlrun[complete]==0.8.0) (0.3)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /usr/local/lib/python3.7/site-packages (from jupyter-client->ipykernel~=5.0->mlrun[complete]==0.8.0) (4.9.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.8.0) (0.4.8)\n",
      "Requirement already satisfied: nbformat in /usr/local/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (5.1.3)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.12.1)\n",
      "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.12.0)\n",
      "Requirement already satisfied: Send2Trash>=1.5.0 in /usr/local/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (1.8.0)\n",
      "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (21.1.0)\n",
      "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.1.2)\n",
      "Requirement already satisfied: bleach in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (4.1.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (1.5.0)\n",
      "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.7.1)\n",
      "Requirement already satisfied: testpath in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.5.0)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.8.4)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /usr/local/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.5.4)\n",
      "Requirement already satisfied: isodate>=0.6.0 in /usr/local/lib/python3.7/site-packages (from msrest>=0.6.21->azure-keyvault-secrets~=4.2; extra == \"complete\"->mlrun[complete]==0.8.0) (0.6.0)\n",
      "Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /usr/local/lib/python3.7/site-packages (from msal<2.0.0,>=1.12.0->azure-identity~=1.5; extra == \"complete\"->mlrun[complete]==0.8.0) (2.3.0)\n",
      "Requirement already satisfied: portalocker~=1.0; platform_system != \"Windows\" in /usr/local/lib/python3.7/site-packages (from msal-extensions~=0.3.0->azure-identity~=1.5; extra == \"complete\"->mlrun[complete]==0.8.0) (1.7.1)\n",
      "Requirement already satisfied: pyparsing<3,>=2.0.2 in /usr/local/lib/python3.7/site-packages (from packaging>=16.8->bokeh~=2.3.0; extra == \"complete\"->mlrun[complete]==0.8.0) (2.4.7)\n",
      "Requirement already satisfied: adal>=0.4.2 in /usr/local/lib/python3.7/site-packages (from azure-datalake-store<0.1,>=0.0.46->adlfs~=2021.8.1; extra == \"complete\"->mlrun[complete]==0.8.0) (1.2.7)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.7/site-packages (from google-resumable-media<3.0dev,>=1.3.0; python_version >= \"3.6\"->google-cloud-storage>=1.13.0->kfp~=1.0.1->mlrun[complete]==0.8.0) (1.3.0)\n",
      "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/site-packages (from bleach->nbconvert>=5.4->nuclio-jupyter~=0.8.21->mlrun[complete]==0.8.0) (0.5.1)\n",
      "WARNING: You are using pip version 20.2.4; however, version 21.3.1 is available.\n",
      "You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
      "\u001b[36mINFO\u001b[0m[0032] Taking snapshot of full filesystem...        \n",
      "\u001b[36mINFO\u001b[0m[0032] Pushing image to docker-registry.default-tenant.app.app-lab-eks.iguazio-cd2.com:80/mlrun/func-nyc-taxi-avia-taxi:latest \n",
      "\u001b[36mINFO\u001b[0m[0033] Pushed image to 1 destinations               \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from mlrun.platforms import auto_mount\n",
    "taxi_func.apply(auto_mount())\n",
    "taxi_func.deploy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:27:48,614 [info] starting run fetch_data uid=dadb9814a93e4238b2ed65ce3712cb46 DB=http://mlrun-api:8080\n",
      "> 2021-11-16 13:27:48,882 [info] Job is running in the background, pod: fetch-data-mnpzt\n",
      "> 2021-11-16 13:27:56,454 [info] Reading taxi records data from https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\n",
      "> 2021-11-16 13:28:01,211 [info] Reading zones data from https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\n",
      "> 2021-11-16 13:28:01,351 [info] Saving datasets to v3io:///projects/nyc-taxi-avia/artifacts/data ...\n",
      "> 2021-11-16 13:28:08,677 [info] run executed, status=completed\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-avia</td>\n",
       "      <td><div title=\"dadb9814a93e4238b2ed65ce3712cb46\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/dadb9814a93e4238b2ed65ce3712cb46/overview\" target=\"_blank\" >...3712cb46</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Nov 16 13:27:56</td>\n",
       "      <td>completed</td>\n",
       "      <td>fetch_data</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=avia</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=avia</div><div class=\"dictlist\">host=fetch-data-mnpzt</div></td>\n",
       "      <td><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\">taxi_records_csv_path</div><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result892e11c6\" title=\"files/v3io/projects/nyc-taxi-avia/artifacts/data/nyc-taxi-dataset.csv\">nyc-taxi-dataset</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result892e11c6\" title=\"files/v3io/projects/nyc-taxi-avia/artifacts/data/zones-dataset.csv\">zones-dataset</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result892e11c6-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result892e11c6-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result892e11c6\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result892e11c6-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/dadb9814a93e4238b2ed65ce3712cb46/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:28:10,664 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "fetch_data_run = taxi_func.run(name='fetch_data',\n",
    "                               handler='fetch_data',\n",
    "                               inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                                       'zones_csv_path': zones_csv_path})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset': 'store://artifacts/nyc-taxi-avia/fetch_data_nyc-taxi-dataset:dadb9814a93e4238b2ed65ce3712cb46',\n",
       " 'zones-dataset': 'store://artifacts/nyc-taxi-avia/fetch_data_zones-dataset:dadb9814a93e4238b2ed65ce3712cb46'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_data_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:28:10,681 [info] starting run transform_dataset uid=832dc31589a1456f9c12fc1b8dce6dae DB=http://mlrun-api:8080\n",
      "> 2021-11-16 13:28:10,908 [info] Job is running in the background, pod: transform-dataset-2kpdv\n",
      "> 2021-11-16 13:28:18,342 [info] Begin datasets transform\n",
      "> 2021-11-16 13:28:18,342 [info] zones_csv_path: v3io:///projects/nyc-taxi-avia/artifacts/data/zones-dataset.csv\n",
      "> 2021-11-16 13:29:12,814 [info] End dataset transform\n",
      "> 2021-11-16 13:29:12,878 [info] run executed, status=completed\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-avia</td>\n",
       "      <td><div title=\"832dc31589a1456f9c12fc1b8dce6dae\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/832dc31589a1456f9c12fc1b8dce6dae/overview\" target=\"_blank\" >...8dce6dae</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Nov 16 13:28:18</td>\n",
       "      <td>completed</td>\n",
       "      <td>transform_dataset</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=avia</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=avia</div><div class=\"dictlist\">host=transform-dataset-2kpdv</div></td>\n",
       "      <td><div title=\"store://artifacts/nyc-taxi-avia/fetch_data_nyc-taxi-dataset:dadb9814a93e4238b2ed65ce3712cb46\">taxi_records_csv_path</div><div title=\"store://artifacts/nyc-taxi-avia/fetch_data_zones-dataset:dadb9814a93e4238b2ed65ce3712cb46\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultbf2d2e46\" title=\"files/v3io/projects/nyc-taxi-avia/artifacts/data/nyc-taxi-dataset-transformed.csv\">nyc-taxi-dataset-transformed</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultbf2d2e46-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultbf2d2e46-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultbf2d2e46\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultbf2d2e46-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/832dc31589a1456f9c12fc1b8dce6dae/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:29:19,885 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "transform_dataset_run = taxi_func.run(name='transform_dataset',\n",
    "                                      handler='transform_dataset',\n",
    "                                      inputs={'taxi_records_csv_path': fetch_data_run.outputs['nyc-taxi-dataset'],\n",
    "                                              'zones_csv_path': fetch_data_run.outputs['zones-dataset']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset-transformed': 'store://artifacts/nyc-taxi-avia/transform_dataset_nyc-taxi-dataset-transformed:832dc31589a1456f9c12fc1b8dce6dae'}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform_dataset_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:29:19,902 [info] starting run train_model uid=e3578a8abdee42b8b0de900598598d8c DB=http://mlrun-api:8080\n",
      "> 2021-11-16 13:29:20,145 [info] Job is running in the background, pod: train-model-j985n\n",
      "> 2021-11-16 13:29:27,510 [info] Begin training\n",
      "> 2021-11-16 13:29:27,510 [info] LGBM version is 3.3.1\n",
      "[LightGBM] [Warning] Met categorical feature which contains sparse values. Consider renumbering to consecutive integers started from zero\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.006805 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 23961\n",
      "[LightGBM] [Info] Number of data points in the train set: 879294, number of used features: 16\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Info] Start training from score 12.418691\n",
      "Training until validation scores don't improve for 500 rounds\n",
      "[500]\tvalid_0's rmse: 3.01302\n",
      "[1000]\tvalid_0's rmse: 2.98593\n",
      "[1500]\tvalid_0's rmse: 2.98292\n",
      "[2000]\tvalid_0's rmse: 2.98664\n",
      "Early stopping, best iteration is:\n",
      "[1551]\tvalid_0's rmse: 2.98226\n",
      "> 2021-11-16 13:30:10,742 [info] End training\n",
      "> 2021-11-16 13:30:10,804 [info] run executed, status=completed\n",
      "Found `num_rounds` in params. Will use it instead of argument\n",
      "'early_stopping_rounds' argument is deprecated and will be removed in a future release of LightGBM. Pass 'early_stopping()' callback via 'callbacks' argument instead.\n",
      "Using categorical_feature in Dataset.\n",
      "'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n",
      "'silent' argument is deprecated and will be removed in a future release of LightGBM. Pass 'verbose' parameter via 'params' instead.\n",
      "Overriding the parameters from Reference Dataset.\n",
      "categorical_column in param dict is overridden.\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-avia</td>\n",
       "      <td><div title=\"e3578a8abdee42b8b0de900598598d8c\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/e3578a8abdee42b8b0de900598598d8c/overview\" target=\"_blank\" >...98598d8c</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Nov 16 13:29:27</td>\n",
       "      <td>completed</td>\n",
       "      <td>train_model</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=avia</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=avia</div><div class=\"dictlist\">host=train-model-j985n</div></td>\n",
       "      <td><div title=\"store://artifacts/nyc-taxi-avia/transform_dataset_nyc-taxi-dataset-transformed:832dc31589a1456f9c12fc1b8dce6dae\">input_ds</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div title=\"v3io:///projects/nyc-taxi-avia/artifacts/models/\">FareModel</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result779ec7a7-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result779ec7a7-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result779ec7a7\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result779ec7a7-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/e3578a8abdee42b8b0de900598598d8c/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:30:12,245 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "train_model_run = taxi_func.run(name='train_model',\n",
    "                                handler='train_model',\n",
    "                                inputs={'input_ds': transform_dataset_run.outputs['nyc-taxi-dataset-transformed']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'FareModel': 'store://artifacts/nyc-taxi-avia/train_model_FareModel:e3578a8abdee42b8b0de900598598d8c'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serving"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model serving class is in model-serving.ipynb."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:30:16,791 [info] Starting remote function deploy\n",
      "2021-11-16 13:30:16  (info) Deploying function\n",
      "2021-11-16 13:30:16  (info) Building\n",
      "2021-11-16 13:30:17  (info) Staging files and preparing base images\n",
      "2021-11-16 13:30:17  (info) Building processor image\n",
      "2021-11-16 13:30:18  (info) Build complete\n",
      "2021-11-16 13:30:27  (info) Function deploy complete\n",
      "> 2021-11-16 13:30:28,107 [info] successfully deployed function: {'internal_invocation_urls': ['nuclio-nyc-taxi-avia-model-serving.default-tenant.svc.cluster.local:8080'], 'external_invocation_urls': ['nyc-taxi-avia-model-serving-nyc-taxi-avia.default-tenant.app.app-lab-eks.iguazio-cd2.com/']}\n"
     ]
    }
   ],
   "source": [
    "serving = mlrun.code_to_function(filename=path.abspath('model-serving.ipynb'),kind = 'serving',image='mlrun/mlrun').apply(auto_mount())\n",
    "\n",
    "serving.spec.default_class = 'LGBMModel'\n",
    "serving.spec.build.commands = ['pip install lightgbm']\n",
    "serving.add_model('taxi-serving', train_model_run.outputs['FareModel'])\n",
    "serving_address = serving.deploy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2021-11-16 13:30:28,195 [info] invoking function: {'method': 'POST', 'path': 'http://nuclio-nyc-taxi-avia-model-serving.default-tenant.svc.cluster.local:8080/v2/models/taxi-serving/predict'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'id': '875829b7-8ab7-4bdc-b999-b20039649b9d',\n",
       " 'model_name': 'taxi-serving',\n",
       " 'outputs': [9.52302976897415]}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_data = '''{\"inputs\":[[1,0.80,0.711950,-1.291073,0.712059,1.290988,13,1,1,1,2019,47.274013,40.386065,16.975747,26.587155,18.925788]]}'''\n",
    "serving.invoke('/v2/models/taxi-serving/predict', my_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kubeflow Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Project Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.serving.ServingRuntime at 0x7f1e6ea25290>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project_path = path.abspath('conf')\n",
    "project = mlrun.new_project(project_name_base,\n",
    "                            context=project_path,\n",
    "                            init_git=True,\n",
    "                            user_project=True)\n",
    "\n",
    "project.set_function(f'db://{project.name}/taxi')\n",
    "project.set_function(f'db://{project.name}/model-serving')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing /User/aviaIguazio/demos/howto/converting-to-mlrun/conf/workflow.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile {path.join(project_path, 'workflow.py')}\n",
    "from kfp import dsl\n",
    "from mlrun.platforms import auto_mount\n",
    "import os\n",
    "import sys\n",
    "import mlrun\n",
    "\n",
    "funcs = {}\n",
    "\n",
    "# init functions is used to configure function resources and local settings\n",
    "def init_functions(functions: dict, project=None, secrets=None):\n",
    "    for f in functions.values():\n",
    "        f.apply(auto_mount())\n",
    "\n",
    "@dsl.pipeline(\n",
    "    name=\"NYC Taxi Demo\",\n",
    "    description=\"Convert ML script to MLRun\"\n",
    ")\n",
    "\n",
    "def kfpipeline():\n",
    "\n",
    "    taxi_records_csv_path = mlrun.get_sample_path('data/Taxi/yellow_tripdata_2019-01_subset.csv')\n",
    "    zones_csv_path = mlrun.get_sample_path('data/Taxi/taxi_zones.csv')\n",
    "    \n",
    "    # build our ingestion function (container image)\n",
    "    builder = funcs['taxi'].deploy_step(skip_deployed=True)\n",
    "    \n",
    "    # run the ingestion function with the new image and params\n",
    "    ingest = funcs['taxi'].as_step(\n",
    "        name=\"fetch_data\",\n",
    "        handler='fetch_data',\n",
    "        image=builder.outputs['image'],\n",
    "        inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                'zones_csv_path': zones_csv_path},\n",
    "        outputs=['nyc-taxi-dataset', 'zones-dataset'])\n",
    "\n",
    "    # Join and transform the data sets \n",
    "    transform = funcs[\"taxi\"].as_step(\n",
    "        name=\"transform_dataset\",\n",
    "        handler='transform_dataset',\n",
    "        inputs={\"taxi_records_csv_path\": ingest.outputs['nyc-taxi-dataset'],\n",
    "                \"zones_csv_path\" : ingest.outputs['zones-dataset']},\n",
    "        outputs=['nyc-taxi-dataset-transformed'])\n",
    "\n",
    "    # Train the model\n",
    "    train = funcs[\"taxi\"].as_step(\n",
    "        name=\"train\",\n",
    "        handler=\"train_model\",\n",
    "        inputs={\"input_ds\" : transform.outputs['nyc-taxi-dataset-transformed']},\n",
    "        outputs=['FareModel'])\n",
    "    \n",
    "    # Deploy the model\n",
    "    deploy = funcs[\"model-serving\"].deploy_step(models={\"taxi-serving_v1\": train.outputs['FareModel']}, tag='v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "project.set_workflow('main', 'workflow.py', embed=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "project.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: kfp Pages: 1 -->\n",
       "<svg width=\"198pt\" height=\"332pt\"\n",
       " viewBox=\"0.00 0.00 197.78 332.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 328)\">\n",
       "<title>kfp</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-328 193.7832,-328 193.7832,4 -4,4\"/>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124</title>\n",
       "<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-90\" rx=\"33.2948\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"94.8916\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">train</text>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3011829852 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3011829852</title>\n",
       "<polygon fill=\"#00ff00\" stroke=\"#000000\" points=\"179.8916,-36 13.8916,-36 9.8916,-32 9.8916,0 175.8916,0 179.8916,-4 179.8916,-36\"/>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 9.8916,-32 \"/>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 175.8916,0 \"/>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 179.8916,-36 \"/>\n",
       "<text text-anchor=\"middle\" x=\"94.8916\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">deploy&#45;model&#45;serving</text>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3011829852 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3011829852</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-71.8314C94.8916,-64.131 94.8916,-54.9743 94.8916,-46.4166\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-46.4132 94.8916,-36.4133 91.3917,-46.4133 98.3917,-46.4132\"/>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3004087308 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3004087308</title>\n",
       "<polygon fill=\"#00ff00\" stroke=\"#000000\" points=\"142.3916,-324 47.3916,-324 47.3916,-288 142.3916,-288 142.3916,-324\"/>\n",
       "<text text-anchor=\"middle\" x=\"94.8916\" y=\"-302.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">deploy&#45;taxi</text>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245</title>\n",
       "<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-234\" rx=\"57.6901\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"94.8916\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">fetch&#45;data</text>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3004087308&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3004087308&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-287.8314C94.8916,-280.131 94.8916,-270.9743 94.8916,-262.4166\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-262.4132 94.8916,-252.4133 91.3917,-262.4133 98.3917,-262.4132\"/>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871</title>\n",
       "<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-162\" rx=\"94.7833\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"94.8916\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">transform&#45;dataset</text>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;3638815245&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-215.8314C94.8916,-208.131 94.8916,-198.9743 94.8916,-190.4166\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-190.4132 94.8916,-180.4133 91.3917,-190.4133 98.3917,-190.4132\"/>\n",
       "</g>\n",
       "<!-- nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>nyc&#45;taxi&#45;demo&#45;lpp6m&#45;661210871&#45;&gt;nyc&#45;taxi&#45;demo&#45;lpp6m&#45;2421845124</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-143.8314C94.8916,-136.131 94.8916,-126.9743 94.8916,-118.4166\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-118.4132 94.8916,-108.4133 91.3917,-118.4133 98.3917,-118.4132\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f1e6cc84cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>Run Results</h2>Workflow 3b308184-a35a-45d1-a81d-3182332ed0ef finished, state=Succeeded<br>click the hyper links below to see detailed results<br><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>uid</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td><div title=\"8271c368075b4fdda8247449954391de\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/8271c368075b4fdda8247449954391de/overview\" target=\"_blank\" >...954391de</a></div></td>\n",
       "      <td>Nov 16 13:32:49</td>\n",
       "      <td>completed</td>\n",
       "      <td>train</td>\n",
       "      <td></td>\n",
       "      <td><div title=\"/User/aviaIguazio/demos/howto/converting-to-mlrun/pipe/3b308184-a35a-45d1-a81d-3182332ed0ef/models\">FareModel</div></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"5434bd5cfc4b475cb53e198c9c8ac761\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/5434bd5cfc4b475cb53e198c9c8ac761/overview\" target=\"_blank\" >...9c8ac761</a></div></td>\n",
       "      <td>Nov 16 13:31:24</td>\n",
       "      <td>completed</td>\n",
       "      <td>transform_dataset</td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result\" title=\"/files/aviaIguazio/demos/howto/converting-to-mlrun/pipe/3b308184-a35a-45d1-a81d-3182332ed0ef/data/nyc-taxi-dataset-transformed.csv\">nyc-taxi-dataset-transformed</div></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"c0ef7578d3424ab3afb2a33924de9251\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks.iguazio-cd2.com/mlprojects/nyc-taxi-avia/jobs/monitor/c0ef7578d3424ab3afb2a33924de9251/overview\" target=\"_blank\" >...24de9251</a></div></td>\n",
       "      <td>Nov 16 13:30:46</td>\n",
       "      <td>completed</td>\n",
       "      <td>fetch_data</td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result\" title=\"/files/aviaIguazio/demos/howto/converting-to-mlrun/pipe/3b308184-a35a-45d1-a81d-3182332ed0ef/data/nyc-taxi-dataset.csv\">nyc-taxi-dataset</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result\" title=\"/files/aviaIguazio/demos/howto/converting-to-mlrun/pipe/3b308184-a35a-45d1-a81d-3182332ed0ef/data/zones-dataset.csv\">zones-dataset</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "artifact_path = path.abspath('./pipe/{{workflow.uid}}')\n",
    "run_id = project.run(\n",
    "    'main',    \n",
    "    artifact_path=artifact_path, \n",
    "    dirty=True,\n",
    "    watch=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}