viz-4a-anomaly-detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Home](https://walterra.github.io/jupyter2kibana/)\n",
    "\n",
    "*To make sure the Vega charts render correctly, view the notebook not from the Github repo but the published website here: \n",
    "https://walterra.github.io/jupyter2kibana/viz-4a-anomaly-detection.html*\n",
    "\n",
    "# viz-4a-anomaly-detection.ipynb\n",
    "\n",
    "The first notebook in this set of examples (viz-4x) demonstrates how to replicate one of the custom charts of Kibana's Machine Learning plugin as an embeddable chart for dashboards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import altair as alt\n",
    "import eland as ed\n",
    "from elasticsearch import Elasticsearch\n",
    "import elastic_transport\n",
    "import logging\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import urllib3\n",
    "import warnings\n",
    "\n",
    "alt.data_transformers.disable_max_rows()\n",
    "logging.getLogger(\"elastic_transport\").setLevel(logging.ERROR)\n",
    "\n",
    "# Suppress insecure SSL connection warnings\n",
    "# In dev environments with the `verify_certs=False`\n",
    "# you might want to reduce those warnings.\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "urllib3.disable_warnings(elastic_transport.SecurityWarning)\n",
    "\n",
    "# For rendering the notebook to HTML hide all warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_name = '.ml-anomalies-fq_single_count_15m'\n",
    "vis_name = 'ml-anomalies-fq'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(975, 73)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('config.json') as config_file:\n",
    "  es_config = json.load(config_file)\n",
    "\n",
    "# First instantiate an 'Elasticsearch' instance with the supplied config\n",
    "es = Elasticsearch(\n",
    "    hosts=[es_config['es_client']],\n",
    "    basic_auth=[\n",
    "        es_config['user'],\n",
    "        es_config['password']\n",
    "    ],\n",
    "    # Only in development environments with self signed certificates fall back to use `verify_certs=False`\n",
    "    verify_certs=False\n",
    ")\n",
    "\n",
    "ed_df = ed.DataFrame(es, index_name)\n",
    "ed_df_number = ed_df.select_dtypes(include=np.number)\n",
    "ed_df_number.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-b89de39d2c6e484aafe6b98949fb344a\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-b89de39d2c6e484aafe6b98949fb344a\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-b89de39d2c6e484aafe6b98949fb344a\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"concat\": [{\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"actual\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'actual')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"anomaly_score\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'anomaly_score')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"event_count\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'event_count')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_lower\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_lower')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_median\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_median')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"model_upper\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'model_upper')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"result_type\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'result_type')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"timestamp\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'timestamp')\"}], \"width\": 100}, {\"mark\": \"bar\", \"encoding\": {\"tooltip\": [{\"bin\": true, \"field\": \"value\", \"title\": \"x\", \"type\": \"quantitative\"}, {\"aggregate\": \"count\", \"title\": \"y\", \"type\": \"quantitative\"}], \"x\": {\"bin\": true, \"field\": \"value\", \"title\": \"\", \"type\": \"quantitative\"}, \"y\": {\"aggregate\": \"count\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 100, \"title\": \"typical\", \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"fold\": [\"anomaly_score\", \"actual\", \"typical\", \"event_count\", \"model_lower\", \"model_median\", \"model_upper\", \"timestamp\", \"result_type\"], \"as\": [\"attribute\", \"value\"]}, {\"filter\": \"(datum.attribute === 'typical')\"}], \"width\": 100}], \"columns\": 4, \"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}}, \"resolve\": {\"axis\": {\"x\": \"independent\", \"y\": \"independent\"}, \"scale\": {\"x\": \"independent\", \"y\": \"independent\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.ConcatChart(...)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: To create the Vega spec using Altair we reference ES via URL first. This will only work\n",
    "# for non-secured ES instances. If your ES instance runs using SSL and/or authentication the chart\n",
    "# in this cell will render empty. You can still save the visualization in Kibana correctly in the\n",
    "# next cell because there the URL gets replaced with an Elasticsearch query\n",
    "# to be used via the Kibana Vega plugin.\n",
    "\n",
    "# WARNING:\n",
    "# Do the following approach using a proxy only for demo purposes in a development environment.\n",
    "# It will expose a secured ES instance unsecured!\n",
    "# To make this work for demo purposes run the nodejs based proxy in a separate terminal like this:\n",
    "# NODE_TLS_REJECT_UNAUTHORIZED='0' node proxy\n",
    "\n",
    "# URL as ES endpoint\n",
    "# url = 'http://localhost:9220/'+index_name+'/_search?size=1000'\n",
    "\n",
    "# URL static fallback\n",
    "url = 'https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json'\n",
    "\n",
    "url_data = alt.Data(url=url, format=alt.DataFormat(property='hits.hits',type='json'))\n",
    "\n",
    "fields = [\n",
    "    'anomaly_score',\n",
    "    'actual',\n",
    "    'typical',\n",
    "    'event_count',\n",
    "    'model_lower',\n",
    "    'model_median',\n",
    "    'model_upper',\n",
    "    'timestamp',\n",
    "    'result_type'\n",
    "]\n",
    "\n",
    "rename_dict = dict((a, 'datum._source.'+a) for a in fields)\n",
    "\n",
    "base = alt.Chart(url_data).transform_calculate(**rename_dict)\n",
    "\n",
    "url_chart = base.transform_fold(\n",
    "    fields,\n",
    "    as_=['attribute', 'value']\n",
    ").mark_bar().encode(\n",
    "    alt.X('value:Q', bin=True, title=''),\n",
    "    alt.Y('count()', title=''),\n",
    "    tooltip=[\n",
    "        alt.Tooltip('value:Q', bin=True, title='x'),\n",
    "        alt.Tooltip('count()', title='y')\n",
    "    ]\n",
    ").properties(\n",
    "    width=100,\n",
    "    height=100\n",
    ")\n",
    "\n",
    "url_charts = alt.ConcatChart(\n",
    "    concat=[\n",
    "      url_chart.transform_filter(alt.datum.attribute == attribute).properties(title=attribute)\n",
    "      for attribute in sorted(fields)\n",
    "    ],\n",
    "    columns=4\n",
    ").resolve_axis(\n",
    "    x='independent',\n",
    "    y='independent'\n",
    ").resolve_scale(\n",
    "    x='independent', \n",
    "    y='independent'\n",
    ")\n",
    "\n",
    "url_charts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-6c0eec20b2ee4d85b35b93b6229ea2de\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-6c0eec20b2ee4d85b35b93b6229ea2de\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-6c0eec20b2ee4d85b35b93b6229ea2de\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300, \"strokeWidth\": 0}, \"axis\": {\"domainColor\": \"#ddd\", \"grid\": true, \"gridColor\": \"#eee\", \"labelBound\": true, \"labelColor\": \"gray\", \"tickColor\": \"#ddd\"}, \"title\": {\"anchor\": \"start\", \"color\": \"gray\", \"fontSize\": 14, \"fontWeight\": \"bold\"}}, \"layer\": [{\"mark\": {\"type\": \"area\", \"color\": \"lightgray\", \"opacity\": 0.5}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"model_upper\", \"type\": \"quantitative\"}, \"y2\": {\"aggregate\": \"mean\", \"field\": \"model_lower\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"line\", \"color\": \"gray\", \"opacity\": 0.5, \"strokeJoin\": \"round\"}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"model_median\", \"title\": \"\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"line\", \"color\": \"#1f77b4\", \"strokeJoin\": \"round\"}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"aggregate\": \"mean\", \"field\": \"actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"filter\": \"(datum.result_type === 'model_plot')\"}], \"width\": 800}, {\"mark\": {\"type\": \"point\", \"color\": \"red\", \"size\": 60}, \"encoding\": {\"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"field\": \"mean_actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"aggregate\": [{\"op\": \"mean\", \"field\": \"actual\", \"as\": \"mean_actual\"}, {\"op\": \"mean\", \"field\": \"anomaly_score\", \"as\": \"mean_anomaly_score\"}], \"groupby\": [\"timestamp\"]}, {\"filter\": \"(datum.mean_anomaly_score > 0)\"}], \"width\": 800}, {\"mark\": {\"type\": \"circle\", \"opacity\": 0, \"size\": 100}, \"encoding\": {\"tooltip\": [{\"field\": \"mean_actual\", \"type\": \"quantitative\"}, {\"field\": \"mean_anomaly_score\", \"type\": \"quantitative\"}, {\"field\": \"timestamp\", \"type\": \"temporal\"}], \"x\": {\"axis\": {\"grid\": false}, \"field\": \"timestamp\", \"scale\": {\"nice\": false, \"zero\": false}, \"title\": \"\", \"type\": \"temporal\"}, \"y\": {\"field\": \"mean_actual\", \"type\": \"quantitative\"}}, \"height\": 300, \"transform\": [{\"calculate\": \"datum._source.anomaly_score\", \"as\": \"anomaly_score\"}, {\"calculate\": \"datum._source.actual\", \"as\": \"actual\"}, {\"calculate\": \"datum._source.typical\", \"as\": \"typical\"}, {\"calculate\": \"datum._source.event_count\", \"as\": \"event_count\"}, {\"calculate\": \"datum._source.model_lower\", \"as\": \"model_lower\"}, {\"calculate\": \"datum._source.model_median\", \"as\": \"model_median\"}, {\"calculate\": \"datum._source.model_upper\", \"as\": \"model_upper\"}, {\"calculate\": \"datum._source.timestamp\", \"as\": \"timestamp\"}, {\"calculate\": \"datum._source.result_type\", \"as\": \"result_type\"}, {\"aggregate\": [{\"op\": \"mean\", \"field\": \"actual\", \"as\": \"mean_actual\"}, {\"op\": \"mean\", \"field\": \"anomaly_score\", \"as\": \"mean_anomaly_score\"}], \"groupby\": [\"timestamp\"]}], \"width\": 800}], \"data\": {\"url\": \"https://walterra.github.io/jupyter2kibana/data/fq_single_count_15m.json\", \"format\": {\"property\": \"hits.hits\", \"type\": \"json\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\"}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.LayerChart(...)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# custom: ['#1f77b4', 'lightgray', 'gray']\n",
    "# ml:     ['#32a7c2', '#c8e6ef', '#bbd7df']\n",
    "\n",
    "colors = {'model_bounds':'lightgray','model_median':'gray','actual':'#1f77b4'}\n",
    "\n",
    "base = alt.Chart(url_data, width=800, height=300).transform_calculate(**rename_dict)\n",
    "\n",
    "model_bounds = base.mark_area(color=colors['model_bounds'], opacity=0.5).transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(model_upper):Q'),\n",
    "    alt.Y2('mean(model_lower):Q'),\n",
    ")\n",
    "\n",
    "model_median = base.mark_line(color=colors['model_median'], opacity=0.5, strokeJoin='round').transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(model_median):Q', title='')\n",
    ")\n",
    "\n",
    "actual = base.mark_line(color=colors['actual'], strokeJoin='round').transform_filter(\n",
    "    alt.datum.result_type == 'model_plot'\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean(actual):Q')\n",
    ")\n",
    "\n",
    "anomalies = base.mark_point(color='red', size=60).transform_aggregate(\n",
    "    mean_actual='mean(actual)',\n",
    "    mean_anomaly_score='mean(anomaly_score)',\n",
    "    groupby=[\"timestamp\"]\n",
    ").transform_filter(\n",
    "    alt.datum.mean_anomaly_score > 0\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean_actual:Q')\n",
    ")\n",
    "\n",
    "tooltip = base.mark_circle(opacity=0, size=100).transform_aggregate(\n",
    "    mean_actual='mean(actual)',\n",
    "    mean_anomaly_score='mean(anomaly_score)',\n",
    "    groupby=[\"timestamp\"]\n",
    ").encode(\n",
    "    alt.X('timestamp:T', scale=alt.Scale(zero=False,nice=False), title='', axis=alt.Axis(grid=False)),\n",
    "    alt.Y('mean_actual:Q'),\n",
    "    tooltip=['mean_actual:Q', 'mean_anomaly_score:Q', 'timestamp:T']\n",
    ")\n",
    "\n",
    "chart = (model_bounds + model_median + actual + anomalies + tooltip).configure_axis(\n",
    "    grid=True,\n",
    "    gridColor=\"#eee\",\n",
    "    domainColor=\"#ddd\",\n",
    "    tickColor=\"#ddd\",\n",
    "    labelColor=\"gray\",\n",
    "    labelBound=True,\n",
    ").configure_view(\n",
    "    strokeWidth=0\n",
    ").configure_title(\n",
    "    fontSize=14,\n",
    "    fontWeight='bold',\n",
    "    anchor='start',\n",
    "    color='gray'\n",
    ")\n",
    "\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [409]>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from kibana_vega_util import saveVegaLiteVis\n",
    "\n",
    "saveVegaLiteVis(\n",
    "    index_name,\n",
    "    vis_name,\n",
    "    chart,\n",
    "    resultSize=1000,\n",
    "    # Only in development environments with self signed certificates fall back to use `verify=False`\n",
    "    verify=False\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env-jupyter2kibana",
   "language": "python",
   "name": "env-jupyter2kibana"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}