diff --git a/forgebox/__init__.py b/forgebox/__init__.py index 5c4105c..976498a 100644 --- a/forgebox/__init__.py +++ b/forgebox/__init__.py @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "1.0.3" diff --git a/forgebox/widgets.py b/forgebox/widgets.py index ce27487..41f6973 100644 --- a/forgebox/widgets.py +++ b/forgebox/widgets.py @@ -6,7 +6,7 @@ # Cell import pandas as pd from .df import PandasDisplay -from .html import list_group, list_group_kv, display +from .html import list_group, list_group_kv from typing import Callable, List, Tuple, Set, Dict, Any from ipywidgets import ( IntSlider, FloatSlider, Text, Textarea, Layout, @@ -17,6 +17,11 @@ ) import json +try: + display(HTML('''''')) +except: + display = print + def display_df(df): display(df) diff --git a/nbs/05_inter_widgets.ipynb b/nbs/05_inter_widgets.ipynb deleted file mode 100644 index 47aca95..0000000 --- a/nbs/05_inter_widgets.ipynb +++ /dev/null @@ -1,1381 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Interactive tools\n", - "> Very helpful interactive tools" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp widgets" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import pandas as pd\n", - "import numpy as np\n", - "from forgebox.df import PandasDisplay\n", - "from forgebox.html import list_group, list_group_kv\n", - "from typing import Callable, List, Tuple, Set, Dict, Any\n", - "from ipywidgets import (\n", - " IntSlider, FloatSlider, Text, Textarea, Layout,\n", - " Output, HBox, VBox, Button,\n", - " Select, SelectMultiple,\n", - " Dropdown, Checkbox,\n", - " IntProgress, HTML, interact, interact_manual\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def display_df(df):display(df)\n", - "\n", - "def search_box(df,columns,manual = False,max_rows = 10,callback = display_df):\n", - " \"\"\"\n", - " create a search box based on dataframe\n", - " df: pandas dataframe\n", - " columns: str, dataframe field name\n", - " manual: bool, search the dataframe on when click the button(manual=True), \n", - " or on keypress reaction to inputbox (manual=False), default False\n", - " max_rows:int, max rows of show result, default 10\n", - " callback: python callable, discribe the action you want to put on \n", - " search result (a filtered dataframe), default is to display the dataframe\n", - " \"\"\"\n", - " from ipywidgets import interact,interact_manual\n", - " from IPython.display import HTML\n", - " \n", - " intera = interact_manual if manual else interact\n", - "\n", - " @intera\n", - " def search(KeyWord = \"\",):\n", - " for col in columns:\n", - " result = df[col].fillna(\"NaN Value\").str.contains(KeyWord)\n", - " if sum(result)>0:\n", - " with PandasDisplay(max_colwidth=0,max_rows=max_rows):\n", - " display(HTML(f\"

\\\"{KeyWord}\\\" matched on column:[{col}]

\"))\n", - " callback(df[result].head(max_rows))\n", - " return \n", - " print(f\"Nothing found on any column on keyword:{KeyWord}\")\n", - " return " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Search text within dataframe columns\n", - "\n", - "We can have any dataframe we like:" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.read_csv(\"~/Downloads/mesh_keywords_202004201804.csv\").head(2000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Put the dataframe and target column names into the ```search_box()```" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3b3e0ea48c51455a8729a3d2f4fcf5b4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(Text(value='', description='KeyWord'), Output()), _dom_classes=('widget-interact',))" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "search_box(df,columns=[\"keyword\",])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If your dataframe is slightly huge, and slow to respond, you can use ```manual=True``` to turn off the auto keypress response" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6d148ae6a3a142fdba1ef48372e38f68", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(Text(value='', description='KeyWord'), Button(description='Run Interact', style=ButtonSt…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "search_box(df,columns=[\"keyword\",],manual=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Interactive pagination" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def paginate(df,page_len = 20):\n", - " \"\"\"\n", - " Paginate dataframe in jupyter notebook interactively\n", - " Like you can flip through the page\n", - " \"\"\"\n", - " from ipywidgets import interact,interact_manual\n", - " from IPython.display import display,HTML\n", - " pages = len(df)//page_len\n", - " @interact\n", - " def preview(page = (0,pages)):\n", - " display(HTML(f\"

page:{page}/{pages}\"))\n", - " end = (page+1)*page_len\n", - " display(df.head(end).tail(page_len))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a378159cd5ba4be5a1f5ebfb1f7725f5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(IntSlider(value=50, description='page'), Output()), _dom_classes=('widget-interact',))" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "paginate(df)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## SingleButton widget" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A single button widget\n", - "\n", - "It's like interactive_manual with callback\n", - "\n", - "You can make an fully interactive featured widgets if there is only one button to get new page\n", - " \n", - "### Example\n", - " \n", - "```python\n", - "@SingleButton(callback=execute_sql, btn_text:\"run command\", btn_style=\"primary\")\n", - "def makeup_some_sql(\n", - " table={\"typing\":list, \"options\":[\"table_a\", \"table_b\"]},\n", - " limit={\"typing\":int, \"max\":1000, \"step\":10, \"default\":100},\n", - "):\n", - " return f\"select * from {table} limit {limit}\"\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def make_hboxes(\n", - " *widgets,\n", - " sections: int=2\n", - ") -> List[HBox]:\n", - " \"\"\"\n", - " Make a list of HBox, with each hbox\n", - " contans {sections} of widgets at most\n", - " widgets: widget from ipywidgets\n", - " sections: int\n", - " \"\"\"\n", - " hbox_list = []\n", - " hbox_inner=[]\n", - " for idx, widget in enumerate(widgets):\n", - " hbox_inner.append(widget)\n", - " if idx%sections==sections-1:\n", - " hbox_list.append(HBox(hbox_inner))\n", - " hbox_inner=[]\n", - " if len(hbox_inner)>0:\n", - " hbox_list.append(HBox(hbox_inner))\n", - " return hbox_list\n", - "\n", - "class SingleButton:\n", - " \"\"\"\n", - " A single button widget\n", - " It's like interactive_manual with callback\n", - " \n", - " ## Example\n", - " \n", - " ```python\n", - " @SingleButton(callback=execute_sql, btn_text:\"run command\", btn_style=\"primary\")\n", - " def makeup_some_sql(\n", - " table={\"typing\":list, \"options\":[\"table_a\", \"table_b\"]},\n", - " limit={\"typing\":int, \"max\":1000, \"step\":10, \"default\":100},\n", - " ):\n", - " return f\"select * from {table} limit {limit}\"\n", - " ```\n", - " \"\"\"\n", - " def __init__(\n", - " self,\n", - " btn_text: str=\"Run\",\n", - " btn_style: str=\"danger\",\n", - " callback: Callable=lambda x:x,\n", - " sections: int=2,\n", - " ):\n", - " \"\"\"\n", - " btn_text: str, text appears on the button\n", - " btn_style: str, one of the [\"danger\", \"warning\", \"info\", \"primary\"]\n", - " \"\"\"\n", - " self.btn_text = btn_text\n", - " self.btn_style = btn_style\n", - " self.callback = callback\n", - " self.sections = sections\n", - " \n", - " def create_slider(\n", - " self, k: str,\n", - " anno: dict):\n", - " \"\"\"\n", - " create int or float slider widget \n", - " \"\"\"\n", - " typing = anno[\"typing\"]\n", - " if typing == int:\n", - " wi = IntSlider(description=k)\n", - " elif typing == float:\n", - " wi = FloatSlider(description=k)\n", - " else:\n", - " raise TypeError(\n", - " f\"typing should be either int or float, not {typing}\")\n", - " if \"default\" in anno:\n", - " wi.value = anno[\"default\"]\n", - " if \"min\" in anno:\n", - " wi.min = anno[\"min\"]\n", - " if \"max\" in anno:\n", - " wi.max = anno[\"max\"]\n", - " if \"step\" in anno:\n", - " wi.step = anno[\"step\"]\n", - " \n", - " return wi\n", - " \n", - " def create_text(self, k, anno):\n", - " \"\"\"\n", - " create text area widget\n", - " \"\"\"\n", - " if \"textarea\" in anno:\n", - " layout = Layout(width=\"100%\", height=\"auto\" )\n", - " wi = Textarea(description=k, layout=layout)\n", - " else:\n", - " wi = Text(description=k)\n", - " if \"default\" in anno:\n", - " wi.value = anno[\"default\"]\n", - " return wi\n", - " \n", - " def create_dropdown(self, k, anno) -> Dropdown:\n", - " atype = anno[\"typing\"]\n", - " wi = Dropdown(description=k, options=anno[\"options\"])\n", - " if \"default\" in anno:\n", - " wi.value = anno[\"default\"]\n", - " return wi\n", - " \n", - " def anno_widgets(self):\n", - " self.controls = dict()\n", - " for k, anno in self.f.__annotations__.items():\n", - " if \"typing\" not in anno:\n", - " raise KeyError(f\"Missing typing data in arg/kwarg: {k}\")\n", - " atype = anno[\"typing\"]\n", - " if atype in [int, float]:\n", - " create_func = self.create_slider\n", - " elif atype == str:\n", - " create_func = self.create_text\n", - " elif atype in [list, tuple, set, dict, List, Tuple, Set, Dict]:\n", - " create_func = self.create_dropdown\n", - " else:\n", - " raise TypeError(f\"type {atype} not found\")\n", - " self.controls.update({k:create_func(k, anno)})\n", - " \n", - " def execute(self, btn):\n", - " \"\"\"\n", - " Execute target function\n", - " \"\"\"\n", - " self.out.clear_output()\n", - " inputs = dict((b.description, b.value) for b in self.controls.values())\n", - " \n", - " with self.out:\n", - " rt = self.target_func(**inputs)\n", - " return rt\n", - " \n", - " def __call__(self, f: Callable):\n", - " \"\"\"\n", - " Use this class as a decorator\n", - " @SingleButton(callback=func)\n", - " def abc(...):\n", - " ...\n", - " \"\"\"\n", - " self.f=f\n", - " self.name = f.__name__\n", - " self.anno_widgets()\n", - " def wrapper(*args,**kwargs):\n", - " rt = f(*args,**kwargs)\n", - " self.callback(rt)\n", - " return rt\n", - " \n", - " self.target_func = wrapper\n", - " self.out = Output()\n", - " self.run_btn = Button(\n", - " description=self.btn_text, button_style=\"danger\")\n", - " self.hboxes = make_hboxes(\n", - " *list(self.controls.values()),\n", - " sections=self.sections)\n", - " self.btn_boxes = make_hboxes(self.run_btn, sections=self.sections)\n", - " \n", - " self.run_btn.on_click(self.execute)\n", - " self.vbox = VBox([*self.hboxes, *self.btn_boxes, self.out])\n", - " display(self.vbox)\n", - " return f" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example for SingleButton" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is a sample callback" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.html import DOM\n", - "def sql_callback(x):\n", - " DOM(x,\"h3\")()\n", - " # and we execute more code like,\n", - " # eg. run this line in sql\n", - " return x" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can have an interactive single button page, with multipule input widgets" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d95b90652b954e1ba7c1ee44256e90e3", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(HBox(children=(IntSlider(value=10, description='limit', max=20, min=5), Text(value='where 1=1',…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@SingleButton(callback=sql_callback)\n", - "def abc(\n", - " limit:{\"typing\":int, \"default\":10, \"min\":5, \"max\":20},\n", - " where_condition:{\"typing\":str, \"default\": \"where 1=1\", },\n", - " table:{\"typing\":list, \"options\":[\"image_urls\",\"patient_data\"]},\n", - "):\n", - " return f\"sql > SELECT * FROM {table} {where_condition} LIMIT {limit}\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Dataframe labeler" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class Labeler:\n", - " \"\"\"\n", - " An interactive tool labeling pandas dataframe\n", - " row by row\n", - " \"\"\"\n", - " def __init__(\n", - " self,\n", - " df: pd.DataFrame,\n", - " options_col: str,\n", - " result_col: str=\"label\",\n", - " show_callback: Callable=None,\n", - " auto_fill_in: bool=True\n", - " ):\n", - " \"\"\"\n", - " - df: pd.DataFrame, a dataframe prepared\n", - " - options_col: str, column name assigned with options\n", - " - result_col: str, default 'label',\n", - " column name we'll assign the labels\n", - " - auto_fill_in: bool, defult True, do we fill in the label\n", - " automatically if there is only one option\n", - " \"\"\"\n", - " self.df = df\n", - " self.result_col = result_col\n", - " self.options_col = options_col\n", - " \n", - " if auto_fill_in:\n", - " self.df.loc[:,self.result_col] = self.df[self.options_col]\\\n", - " .apply(lambda x:x[0] if len(x)==1 else None)\n", - " else:\n", - " self.df.loc[:,self.result_col] = [None,]*len(self.df)\n", - " \n", - " self.out = Output()\n", - " show_dict = lambda idx,row:str(dict(row))\n", - " self.show_callback = show_dict if show_callback is None else show_callback\n", - " \n", - " def get_progress_bar(self, idx) -> HBox:\n", - " \"\"\"\n", - " get a progress bar based on the row index\n", - " \"\"\"\n", - " i = self.df.index.get_loc(idx)\n", - " return HBox([\n", - " IntProgress(value=i, min=0, max=len(self.df)),\n", - " HTML(f\"{i+1}/{len(self.df)}\")\n", - " ])\n", - " \n", - " def button_box(\n", - " self,\n", - " idx: int,\n", - " options: List[str]\n", - " ) -> HBox:\n", - " \"\"\"\n", - " create a box of buttons based on options\n", - " \"\"\"\n", - " btns = []\n", - " for o in options:\n", - " btn = Button(description = o,)\n", - " btn.value=o\n", - " btn.idx=idx\n", - " btn.on_click(self.choice_cb)\n", - " btns.append(btn)\n", - " return HBox(btns)\n", - " \n", - " def choice_cb(self, option):\n", - " with self.out:\n", - " self.df.loc[option.idx, self.result_col] = option.value\n", - " try:\n", - " idx,row = next(self.gen)\n", - " except StopIteration as e:\n", - " self.out.clear_output()\n", - " display(self.get_progress_bar(self.df.index[-1]))\n", - " display(HTML(\"

All Done

\"))\n", - " display(HTML(\"
Thanks for all the labeling, now try self.df to see the result
\"))\n", - " return\n", - " self.show_func(idx,row)\n", - " \n", - " def __iter__(self):\n", - " for idx, row in self.df.iterrows():\n", - " if (row[self.result_col] is None) and (len(row[self.options_col]))>1:\n", - " yield idx, row\n", - " \n", - " def __call__(self) -> None:\n", - " \"\"\"\n", - " Start labeling\n", - " \"\"\"\n", - " self.gen = iter(self)\n", - " self.show_func(*next(self.gen))\n", - " display(self.out)\n", - " \n", - " def show_func(self, idx, row) -> None:\n", - " self.out.clear_output()\n", - " with self.out:\n", - " display(\n", - " VBox([\n", - " self.get_progress_bar(idx),\n", - " HTML(self.show_callback(idx, row)),\n", - " self.button_box(idx, row[self.options_col])\n", - " ]))" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
keyword
0Appetite Regulation
1Hymecromone
2Round Ligament of Uterus
3Sugar Acids
4Nogalamycin
\n", - "
" - ], - "text/plain": [ - " keyword\n", - "0 Appetite Regulation\n", - "1 Hymecromone\n", - "2 Round Ligament of Uterus\n", - "3 Sugar Acids\n", - "4 Nogalamycin" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = df.loc[1:].reset_index(drop=True)\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Design a dataframe with a column assigned with options" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
keywordlengthoptions
0Appetite Regulation19[A, B]
1Hymecromone11[A, B]
2Round Ligament of Uterus24[A, B]
3Sugar Acids11[A, B]
4Nogalamycin11[A, B]
............
1994Alcoholics Anonymous20[A, B]
1995Lymphocyte Cooperation22[A, B]
1996Anthocerotophyta16[A, B]
1997Traction8[A, B]
1998H(+)-K(+)-Exchanging ATPase27[A, B]
\n", - "

1999 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " keyword length options\n", - "0 Appetite Regulation 19 [A, B]\n", - "1 Hymecromone 11 [A, B]\n", - "2 Round Ligament of Uterus 24 [A, B]\n", - "3 Sugar Acids 11 [A, B]\n", - "4 Nogalamycin 11 [A, B]\n", - "... ... ... ...\n", - "1994 Alcoholics Anonymous 20 [A, B]\n", - "1995 Lymphocyte Cooperation 22 [A, B]\n", - "1996 Anthocerotophyta 16 [A, B]\n", - "1997 Traction 8 [A, B]\n", - "1998 H(+)-K(+)-Exchanging ATPase 27 [A, B]\n", - "\n", - "[1999 rows x 3 columns]" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.loc[:,\"length\"] = df.keyword.apply(len)\n", - "df[\"options\"] = pd.Series([[\"A\",\"B\"]]*len(df))\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "def show_keyword(idx: int, row: Dict[str, str]):\n", - " kw = row[\"keyword\"]\n", - " length = row[\"length\"]\n", - " return f\"\"\"\n", - "
Keyword\n", - " {kw}\n", - " (with length:{length})\n", - "
\"\"\"\n", - "\n", - "lbl = Labeler(df, options_col=\"options\", show_callback=show_keyword)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Start labeling " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lbl()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\"image.png\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Review the label result" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
keywordlengthoptionslabel
0Appetite Regulation19[A, B]A
1Hymecromone11[A, B]B
2Round Ligament of Uterus24[A, B]A
3Sugar Acids11[A, B]B
4Nogalamycin11[A, B]B
\n", - "
" - ], - "text/plain": [ - " keyword length options label\n", - "0 Appetite Regulation 19 [A, B] A\n", - "1 Hymecromone 11 [A, B] B\n", - "2 Round Ligament of Uterus 24 [A, B] A\n", - "3 Sugar Acids 11 [A, B] B\n", - "4 Nogalamycin 11 [A, B] B" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "lbl.df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Editable list and editable dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "total_width = Layout(width=\"100%\")\n", - "\n", - "\n", - "class EditableList(VBox):\n", - " \"\"\"\n", - " Interactive list\n", - " You can add item to the list\n", - " Each added item has a remove button to remove such item\n", - " \"\"\"\n", - "\n", - " def __init__(self, data_list: List[Any] = [], pretty_json: bool = True):\n", - " super().__init__([], layout=total_width)\n", - " self.pretty_json = pretty_json\n", - " for data in data_list:\n", - " self+data\n", - "\n", - " def create_line(self, data):\n", - " children = list(self.children)\n", - " children.append(self.new_line(data))\n", - " self.children = children\n", - "\n", - " def data_to_dom(self, data):\n", - " if self.pretty_json:\n", - " pretty = list_group_kv(data) if hasattr(\n", - " data, \"keys\") else list_group(data)\n", - " return HTML(str(pretty), layout=total_width)\n", - " else:\n", - " return HTML(json.dumps(data))\n", - "\n", - " def new_line(self, data) -> HBox:\n", - " del_btn = Button(description=\"Remove\", icon=\"trash\")\n", - " del_btn.button_style = 'danger'\n", - " hbox = HBox([del_btn, self.data_to_dom(data)],\n", - " layout=total_width, box_style='info')\n", - " hbox.data = data\n", - "\n", - " def remove_hbox():\n", - " children = list(self.children)\n", - " for i, c in enumerate(children):\n", - " if id(c) == id(hbox):\n", - " children.remove(c)\n", - " self.children = children\n", - " del_btn.click = remove_hbox\n", - " return hbox\n", - "\n", - " def __add__(self, data):\n", - " self.create_line(data)\n", - " return self\n", - "\n", - " def get_data(self) -> List[Any]:\n", - " \"\"\"\n", - " Return the data of this list\n", - " \"\"\"\n", - " return list(x.data for x in self.children)\n", - "\n", - "\n", - "class EditableDict(VBox):\n", - " \"\"\"\n", - " Interactive dictionary\n", - " You can add item to the dictionary\n", - " Each added item has a remove button to remove such item\n", - " \"\"\"\n", - " def __init__(self, data_dict: Dict[str, Any] = dict(), pretty_json: bool = True):\n", - " super().__init__([], layout=total_width)\n", - " self.pretty_json = pretty_json\n", - " self+data_dict\n", - " \n", - " def on_update(self, func):\n", - " \"\"\"\n", - " A decorator to set a function\n", - " Every time the dict changed\n", - " Will execute this function\n", - " the default arg is the dictionary data\n", - " \"\"\"\n", - " self.update_func = func\n", - " return func\n", - " \n", - " def run_update(self):\n", - " if hasattr(self, \"update_func\"):\n", - " self.update_func(self.get_data())\n", - " \n", - " def create_line(self, key: str, data: Any):\n", - " children_map = dict((child.key, child) for child in self.children)\n", - " children_map[key] = self.new_line(key, data)\n", - " self.children = list(children_map.values())\n", - " self.run_update()\n", - " \n", - " def data_to_dom(self, data):\n", - " if self.pretty_json:\n", - " pretty = list_group_kv(data) if hasattr(\n", - " data, \"keys\") else list_group(data)\n", - " return HTML(str(pretty), layout=total_width)\n", - " else:\n", - " return HTML(json.dumps(data))\n", - "\n", - " def new_line(self, key: str, data: Any) -> HBox:\n", - " del_btn = Button(description=\"Remove\", icon=\"trash\")\n", - " del_btn.button_style = 'danger'\n", - " key_info = HTML(f\"

{key}

\")\n", - " hbox = HBox([VBox([key_info, del_btn]), self.data_to_dom(data)],\n", - " layout=total_width, box_style='')\n", - " hbox.data = data\n", - " hbox.key = key\n", - "\n", - " def remove_hbox():\n", - " children = list(self.children)\n", - " for c in children:\n", - " if id(c) == id(hbox):\n", - " children.remove(c)\n", - " self.children = children\n", - " self.run_update()\n", - " del_btn.click = remove_hbox\n", - " return hbox\n", - " \n", - " def __setitem__(self, k, v):\n", - " self.create_line(k, v)\n", - " \n", - " def __add__(self, kv):\n", - " for k,v in kv.items():\n", - " self.create_line(k, v)\n", - " return self\n", - "\n", - " def get_data(self) -> Dict[str, Any]:\n", - " \"\"\"\n", - " Return the data of this dict\n", - " \"\"\"\n", - " return dict((x.key,x.data) for x in self.children)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step by step" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class LivingStep:\n", - " \"\"\"\n", - " A step interactive for StepByStep\n", - " \"\"\"\n", - "\n", - " def __init__(\n", - " self, func: Callable,\n", - " top_block: HTML = None\n", - " ):\n", - " self.output = Output()\n", - " self.func = func\n", - " self.top_block = top_block\n", - "\n", - " def __call__(self, **kwargs):\n", - " with self.output:\n", - " if self.top_block is not None:\n", - " display(self.top_block)\n", - " return self.func(**kwargs)\n", - "\n", - " def new_top_block(self, top_block):\n", - " self.top_block = top_block\n", - "\n", - "\n", - "class StepByStep:\n", - " \"\"\"\n", - " A tool to manage progress step by step\n", - " \"\"\"\n", - "\n", - " def __init__(\n", - " self,\n", - " funcs: Dict[str, Callable],\n", - " top_board: HTML = None,\n", - " kwargs: Dict[str, Any] = dict()\n", - " ):\n", - " self.step_keys: List[str] = list(funcs.keys())\n", - " self.steps: Dict[str, LivingStep] = dict(\n", - " (k, LivingStep(f)) for k, f in funcs.items())\n", - " self.furthest: int = 0\n", - " self.current: int = -1\n", - " self.kwargs: Dict[str, Any] = kwargs\n", - " self.execute_cache: Dict[str, bool] = dict()\n", - " self.top_board: HTML = top_board\n", - " self.page_output: Output = Output()\n", - " self.footer: Output = Output()\n", - " self.create_widget()\n", - "\n", - " def rerun(self,**kwargs):\n", - " \"\"\"\n", - " Rerun the current step function\n", - " \"\"\"\n", - " # find the step\n", - " step: LivingStep = self.steps[self.step_keys[self.current]]\n", - " # clear old output\n", - " step.output.clear_output()\n", - " self.kwargs.update(kwargs)\n", - " step(progress=self, **self.kwargs)\n", - "\n", - " def create_control_bar(self,):\n", - " self.bar_hbox = list()\n", - " self.next_btn: Button = Button(\n", - " description=\"Next\", icon='check', button_style='info')\n", - " self.rerun_btn = Button(description=\"Rerun Step\",\n", - " icon='play', button_style='success')\n", - " self.title = HTML(f\"

Step By Step

\")\n", - " self.next_btn.click = self.next_step\n", - " self.rerun_btn.click = self.rerun\n", - " self.bar_hbox.append(self.title)\n", - " self.bar_hbox.append(self.next_btn)\n", - " self.bar_hbox.append(self.rerun_btn)\n", - " return HBox(self.bar_hbox)\n", - "\n", - " def create_widget(self) -> None:\n", - " self.vbox_list = []\n", - " if self.top_board is not None:\n", - " self.vbox_list.append(self.top_board)\n", - " \n", - " # create buttons for progress axis\n", - " self.progress_btns = dict(\n", - " (k, Button(\n", - " description=f\"{i+1}:{k}\",\n", - " icon=\"cube\",\n", - " button_style=\"danger\"\n", - " if i <= self.furthest else \"\"))\n", - " for i, (k, v) in enumerate(self.steps.items())\n", - " )\n", - " # assign action to first button\n", - " first_btn: Button = list(self.progress_btns.values())[0]\n", - " first_btn.click: Callable = self.to_page_action(0)\n", - " self.progress_bar = HBox(list(self.progress_btns.values()))\n", - " \n", - " # assemble the entire widget\n", - " self.vbox_list.append(self.progress_bar)\n", - " self.vbox_list.append(self.create_control_bar())\n", - " self.vbox_list.append(self.page_output)\n", - " self.widget = VBox(self.vbox_list)\n", - "\n", - " def to_page_action(\n", - " self, page_id: int\n", - " ) -> Callable:\n", - " \"\"\"\n", - " generate the button click function\n", - " \"\"\"\n", - " def to_page_func():\n", - " return self[page_id]\n", - " return to_page_func\n", - "\n", - " def update_furthest(self):\n", - " \"\"\"\n", - " Update the \"furthest mark\"\n", - " Also enact the next progress button\n", - " \"\"\"\n", - " if self.furthest < self.current:\n", - " if self.current < len(self):\n", - " # update even button\n", - " btn = self.progress_btns[self.step_keys[self.current]]\n", - " btn.click = self.to_page_action(\n", - " self.current)\n", - " btn.button_style = 'danger'\n", - " self.furthest = self.current\n", - " \n", - " def __repr__(self):\n", - " keys = \" => \".join(self.step_keys)\n", - " return f\"Progress Axis: [{keys}]\"\n", - "\n", - " def __getitem__(self, page_id):\n", - " \"\"\"\n", - " Display a single page\n", - " \"\"\"\n", - " if (page_id < 0) or (page_id >= len(self)):\n", - " return\n", - " self.current: int = page_id\n", - " key: str = self.step_keys[page_id]\n", - " step: LivingStep = self.steps[key]\n", - " self.title.value: str = f\"

Step {page_id+1}: {key}

\"\n", - " self.page_output.clear_output()\n", - "\n", - " with self.page_output:\n", - " display(step.output)\n", - " if key not in self.execute_cache:\n", - " rt = step(progress=self, **self.kwargs)\n", - " if hasattr(rt,\"keys\"):\n", - " self.kwargs(rt)\n", - " self.execute_cache[key] = True\n", - "\n", - " def next_step(self, **kwargs):\n", - " self.current += 1\n", - " if self.current >= len(self):\n", - " self.current = 0\n", - " self.update_furthest()\n", - " return self[self.current]\n", - "\n", - " def __len__(self):\n", - " return len(self.step_keys)\n", - "\n", - " def __call__(self, **kwargs):\n", - " \"\"\"\n", - " Start the entire progress widget\n", - " \"\"\"\n", - " display(self.widget)\n", - " display(self.footer)\n", - " self.kwargs.update(kwargs)\n", - " self.next_step(**self.kwargs)\n", - "\n", - " def show_in(self, step_name: str) -> Callable:\n", - " \"\"\"\n", - " A decorator that will make the function\n", - " to show under a specific step window\n", - " \"\"\"\n", - " step = self.steps[step_name]\n", - "\n", - " def decorator(func: Callable) -> Callable:\n", - " def wrapper(*args, **kwargs):\n", - " with step.output:\n", - " return func(*args, **kwargs)\n", - " return wrapper\n", - " return decorator\n", - "\n", - " def show_footer(self, func: Callable):\n", - " \"\"\"\n", - " A decorator, where functions excuted\n", - " within this, will be showon under footer\n", - " \"\"\"\n", - " def wrapper(*args, **kwargs):\n", - " with self.footer:\n", - " return func(*args, **kwargs)\n", - " return wrapper" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "def test_step(**kwargs):\n", - " @interact\n", - " def show_step_(how_much=[1,2,4,8,16], charactor = ['🍺', '🏺', '🍶','🍷']):\n", - " return pd.DataFrame({\"stuff\":[charactor*how_much,]*how_much})" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "sbs = StepByStep(\n", - " {\"step1\":test_step,\"some_step\":test_step, \"another_step\":test_step},\n", - " HTML(\"\"\"\n", - "

Detail Steps

\n", - " \"\"\")\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "44d422d4246a456fa273f7bd7a683523", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(HTML(value='\\n

Detail Steps

\\n '), HBox(children=(Button(button_style='danger', d…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d5e759edcf114dabb2772dfb2ea7bbc1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sbs()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/06_spacy.ipynb b/nbs/06_spacy.ipynb deleted file mode 100644 index 2cdbbcf..0000000 --- a/nbs/06_spacy.ipynb +++ /dev/null @@ -1,477 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# spacy toolkit\n", - "> Tools enhancing spacy usage" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp spacy" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from forgebox.html import DOM,JS\n", - "import numpy as np\n", - "import json" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Normalization and cosine distance calculation, between each word of 2 sentences" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def l2norm(x):return np.sqrt(np.power(x,2).sum(-1))\n", - "def normal(x):return (x/l2norm(x)[:,None])\n", - "def distance(a,b):return 1- normal(a)@(normal(b).T)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "\n", - "highlight = \"\"\"\n", - "$(document).ready(function(){\n", - " const red=(dom)=>{\n", - " $(dom)\n", - " .css(\"background-color\",\"#FFCCEE\")\n", - " .css(\"box-shadow\",\"0px 0px 10px #FFCCEE\")\n", - " }\n", - " const white=(dom)=>{\n", - " $(dom).css(\"background-color\",\"#FFFFFF\")\n", - " .css(\"box-shadow\",\"0px 0px 0px #FFFFFF\")\n", - " }\n", - " $(\".nlp_tok\")\n", - " .css(\"margin\",\"1px 2px\")\n", - " .css(\"padding\",\"1px 1px\")\n", - " .css(\"border-radius\",\"5px\");\n", - " var edge = JSON.parse(window.edge_data)\n", - " var edge2 = JSON.parse(window.edge_data2)\n", - " var all_b = []\n", - " for(var a in edge)\n", - " {\n", - " $(`#map_word_src_${a}`).data(\"match\",edge[a].idx)\n", - " $(`#map_word_src_${a}`).css(\"font-weight\",900)\n", - " $(`#map_word_src_${a}`)\n", - " .hover(function(){\n", - " red(this)\n", - " var b_match = $(this).data(\"match\")\n", - " for(var i in b_match){\n", - " var b = b_match[i]\n", - " red(document.querySelector(`#map_word_tgt_${b}`))\n", - " }\n", - " })\n", - " $(`#map_word_src_${a}`)\n", - " .mouseleave(function(){\n", - " white(this)\n", - " var b_match = $(this).data(\"match\")\n", - " for(var i in b_match){\n", - " var b = b_match[i]\n", - " white(document.querySelector(`#map_word_tgt_${b}`))\n", - " }\n", - " })\n", - " }\n", - " for(var a in edge2)\n", - " {\n", - " $(`#map_word_tgt_${a}`).data(\"match\",edge2[a].idx)\n", - " $(`#map_word_tgt_${a}`).css(\"font-weight\",900)\n", - " $(`#map_word_tgt_${a}`)\n", - " .hover(function(){\n", - " red(this)\n", - " var b_match = $(this).data(\"match\")\n", - " for(var i in b_match){\n", - " var b = b_match[i]\n", - " red(document.querySelector(`#map_word_src_${b}`))\n", - " }\n", - " })\n", - " $(`#map_word_tgt_${a}`)\n", - " .mouseleave(function(){\n", - " white(this)\n", - " var b_match = $(this).data(\"match\")\n", - " for(var i in b_match){\n", - " var b = b_match[i]\n", - " white(document.querySelector(`#map_word_src_${b}`))\n", - " }\n", - " })\n", - " }\n", - " \n", - "})\n", - "\"\"\"\n", - "def make_map(distance_map,th = .1):\n", - " \"\"\"\n", - " create a hot spot map between similar token\n", - " return dict, token index with matching token index in the target sentence\n", - " \"\"\"\n", - " a2b_dict = dict()\n", - " tgt_range = np.arange(distance_map.shape[-1])\n", - " for i in range(len(distance_map)):\n", - " slice_ = (distance_map[i,:]0:\n", - " a2b_dict[str(i)]= dict(dist = list(distance_map[i,:][slice_].astype(float)),\n", - " idx = list(tgt_range[slice_].astype(str)))\n", - " return a2b_dict\n", - "\n", - "def make_span(tok,tok_id,classes):\n", - " \"\"\"\n", - " create span for token\n", - " tok,spacy token\n", - " tok_id, span's html id\n", - " \n", - " return forgebox.html.DOM\n", - " \"\"\"\n", - " return DOM(tok.text,\"span\",{\"class\":\" \".join(classes),\"id\":tok_id})\n", - "\n", - "def make_div(doc,id_list,classes_list):\n", - " \"\"\"\n", - " create div tag for a sentence\n", - " doc:spacy doc\n", - " id_list, a list of html id\n", - " classes_list, a list of classes assigning to the token span\n", - " \n", - " return forgebox.html.DOM\n", - " \"\"\"\n", - " div = DOM(\"\",\"div\",{\"class\":\"text-block\"})\n", - " for tok,tok_id,classes in zip(doc,id_list,classes_list):\n", - " div.append(make_span(tok,tok_id,classes))\n", - " return div\n", - "\n", - "def doc2div(doc,distance_map,th=.1,is_src= True):\n", - " totallen = len(doc)\n", - " src_tgt = \"src\" if is_src else \"tgt\"\n", - " id_list = list(f\"map_word_{src_tgt}_{i}\" for i in range(len(doc)))\n", - " if is_src:\n", - " edge_map = make_map(distance_map,th=th)\n", - " DOM(f\"window.edge_data = '{json.dumps(edge_map)}'\",\"script\")()\n", - " else:\n", - " edge_map2 = make_map(distance_map.T,th=th)\n", - " DOM(f\"window.edge_data2 = '{json.dumps(edge_map2)}'\",\"script\")()\n", - " \n", - " return make_div(doc,id_list,zip([src_tgt,]*totallen,[\"nlp_tok\"]*totallen))\n", - "\n", - "\n", - "\n", - "def compare_sentences(A,B,nlp,th=.1):\n", - " \"\"\"\n", - " Compare between sentences\n", - " A: str, a sentence\n", - " B: str, another sentence\n", - " nlp: loaded spacy: eg. from spacy import load;nlp = load(\"some_model_name\"), \n", - " see spacy documentation https://spacy.io/usage/spacy-101/\n", - " \n", - " th:float, threshold for consine distance, smaller it is, fewer matching tokens will be found, \n", - " but assume that means higher quality\n", - " \"\"\"\n", - " doc_a = nlp(A)\n", - " doc_b = nlp(B)\n", - " distance_map = distance(doc_a.tensor,doc_b.tensor)\n", - " DOM(\"Sentence A\",\"h3\",{\"class\":\"text-primary\"})()\n", - " doc2div(doc_a,distance_map,th=th,is_src=True)()\n", - " DOM(\"Sentence B\",\"h3\",{\"class\":\"text-primary\"})()\n", - " doc2div(doc_b,distance_map,th=th,is_src=False)()\n", - " JS(highlight)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Visualizing Sentence Compare" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example\n", - "* Suppose you have 2 sentences, name them A and B" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "A = \"\"\"BACKGROUND: In the COU-AA-302 study (NCT00887198), abiraterone acetate plus prednisone (AAP) \n", - "significantly improved outcomes in patients with metastatic castration-resistant prostate cancer (mCRPC) \n", - "versus prednisone alone. Baseline clinical parameters predicting that treatment response could help inform \n", - "clinical decisions were explored. OBJECTIVE: To identify patients who derive the greatest clinical benefit \n", - "from AAP treatment. DESIGN, SETTING, AND PARTICIPANTS: A total of 1088 mCRPC patients treated with either \n", - "AAP or prednisone in the first-line setting in COU-AA-302 were included in this post hoc analysis. \n", - "INTERVENTION: Abiraterone acetate1000mg daily versus placebo, both plus prednisone 10mg daily. \n", - "OUTCOME MEASUREMENTS AND STATISTICAL ANALYSIS: Univariate and multivariable Cox regression \n", - "analyses were performed, including clinical and pathological parameters for the primary end points overall \n", - "survival (OS) and radiographic progression-free survival (rPFS), and secondary study end points. \n", - "Tumor-associated baseline parameters independently impacting OS were applied to stratify patients according \n", - "to possible treatment effects. RESULTS AND LIMITATIONS: Baseline prostate-specific antigen (PSA), tumor-related \n", - "pain as assessed by the Brief Pain Inventory-Short Form (BPI-SF), and Gleason score (GS) at primary diagnosis\n", - "were identified as tumor-associated variables that independently impacted OS. AAP significantly improved \n", - "outcomes versus prednisone in both group 1 (BPI-SF 0-1 and PSA <80 ng/ml and GS <8; p=0.006; hazard ratio \n", - "[HR]: 0.61) and group 2 (BPI-SF 2-3 and/or PSA ≥80 ng/ml and/or GS ≥8; p=0.03; HR: 0.84). \n", - "The differences observed for treatment effects between groups 1 and 2 for OS (HR: 0.61 vs 0.84), rPFS \n", - "(HR: 0.41 vs 0.59), and time to chemotherapy (HR: 0.64 vs 0.71) were not statistically significant. \n", - "CONCLUSIONS: AAP significantly improved outcomes in mCRPC patients compared with prednisone alone regardless \n", - "of baseline pain and PSA level, and GS at primary diagnosis with no significant differences between observed \n", - "treatment effects in groups 1 and 2. PATIENT SUMMARY: Treatment with abiraterone acetate and prednisone \n", - "(compared with treatment with prednisone only) for metastatic castration-resistant prostate cancer \n", - "increased survival in all patients in the study regardless of pain, prostate-specific antigen levels \n", - "at the start of treatment, and Gleason score at primary diagnosis.\"\"\"\n", - "\n", - "B = \"\"\"'Treatment with abiraterone acetate and prednisone (compared with treatment with prednisone only) \n", - "for metastatic castration-resistant prostate cancer increased survival in all patients in the study regardless \n", - "of pain, prostate-specific antigen levels at the start of treatment, and Gleason score at primary diagnosis.'\n", - "\"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can run the comparation between A & B" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "

Sentence A

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
BACKGROUND:IntheCOU-AA-302study(NCT00887198),abirateroneacetateplusprednisone(AAP)\n", - "significantlyimprovedoutcomesinpatientswithmetastaticcastration-resistantprostatecancer(mCRPC)\n", - "versusprednisonealone.Baselineclinicalparameterspredictingthattreatmentresponsecouldhelpinform\n", - "clinicaldecisionswereexplored.OBJECTIVE:Toidentifypatientswhoderivethegreatestclinicalbenefit\n", - "fromAAPtreatment.DESIGN,SETTING,ANDPARTICIPANTS:Atotalof1088mCRPCpatientstreatedwitheither\n", - "AAPorprednisoneinthefirst-linesettinginCOU-AA-302wereincludedinthisposthocanalysis.\n", - "INTERVENTION:Abirateroneacetate1000mgdailyversusplacebo,bothplusprednisone10mgdaily.\n", - "OUTCOMEMEASUREMENTSANDSTATISTICALANALYSIS:UnivariateandmultivariableCoxregression\n", - "analyseswereperformed,includingclinicalandpathologicalparametersfortheprimaryendpointsoverall\n", - "survival(OS)andradiographicprogression-freesurvival(rPFS),andsecondarystudyendpoints.\n", - "Tumor-associatedbaselineparametersindependentlyimpactingOSwereappliedtostratifypatientsaccording\n", - "topossibletreatmenteffects.RESULTSANDLIMITATIONS:Baselineprostate-specificantigen(PSA),tumor-related\n", - "painasassessedbytheBriefPainInventory-ShortForm(BPI-SF),andGleasonscore(GS)atprimarydiagnosis\n", - "wereidentifiedastumor-associatedvariablesthatindependentlyimpactedOS.AAPsignificantlyimproved\n", - "outcomesversusprednisoneinbothgroup1(BPI-SF0-1andPSA<80ng/mlandGS<8;p=0.006;hazardratio\n", - "[HR]:0.61)andgroup2(BPI-SF2-3and/orPSA≥80ng/mland/orGS≥8;p=0.03;HR:0.84).\n", - "Thedifferencesobservedfortreatmenteffectsbetweengroups1and2forOS(HR:0.61vs0.84),rPFS\n", - "(HR:0.41vs0.59),andtimetochemotherapy(HR:0.64vs0.71)werenotstatisticallysignificant.\n", - "CONCLUSIONS:AAPsignificantlyimprovedoutcomesinmCRPCpatientscomparedwithprednisonealoneregardless\n", - "ofbaselinepainandPSAlevel,andGSatprimarydiagnosiswithnosignificantdifferencesbetweenobserved\n", - "treatmenteffectsingroups1and2.PATIENTSUMMARY:Treatmentwithabirateroneacetateandprednisone\n", - "(comparedwithtreatmentwithprednisoneonly)formetastaticcastration-resistantprostatecancer\n", - "increasedsurvivalinallpatientsinthestudyregardlessofpain,prostate-specificantigenlevels\n", - "atthestartoftreatment,andGleasonscoreatprimarydiagnosis.
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "

Sentence B

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
'Treatmentwithabirateroneacetateandprednisone(comparedwithtreatmentwithprednisoneonly)\n", - "formetastaticcastration-resistantprostatecancerincreasedsurvivalinallpatientsinthestudyregardless\n", - "ofpain,prostate-specificantigenlevelsatthestartoftreatment,andGleasonscoreatprimarydiagnosis.'\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from spacy import load\n", - "nlp = load(\"en_core_web_sm\")\n", - "\n", - "compare_sentences(A,B,nlp)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/09_multiprocess.ipynb b/nbs/09_multiprocess.ipynb deleted file mode 100644 index 69340ab..0000000 --- a/nbs/09_multiprocess.ipynb +++ /dev/null @@ -1,346 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multiprocessing Helper\n", - "> Things to smooth up the multiprocessing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp multiproc" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from pathlib import Path\n", - "import logging\n", - "from typing import List, Dict, Callable, Any, Tuple\n", - "import random\n", - "from multiprocessing import Process" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Separate Process Magic" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class VipaClass:\n", - " \"\"\"\n", - " From meditation word Vipassana\n", - " Where creating a magic function\n", - " you can run a cell without holding the entire notebook\n", - " ```ipython\n", - " %%vipas\n", - " ...do things\n", - " ```\n", - " \"\"\"\n", - " def __init__(self):\n", - " self.procs = []\n", - "\n", - " def vipas(self, line, cell):\n", - " \"\"\"\n", - " run a cell magic function, that will not hold the process\n", - " \"\"\"\n", - " ishell = get_ipython()\n", - " proc = Process(target=ishell.run_cell, args = (cell,), daemon=True)\n", - " proc.start()\n", - " self.procs.append(proc)\n", - " \n", - "try:\n", - " ishell = get_ipython()\n", - " from IPython.core import magic\n", - " vipa_class = VipaClass()\n", - " magic.register_cell_magic(vipa_class.vipas)\n", - " vipas = vipa_class.vipas\n", - "except:\n", - " pass" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read single file line by line" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class SingleFileLiner:\n", - " \"\"\"\n", - " Text data reading line by line for multiprocessing\n", - " \"\"\"\n", - "\n", - " def __init__(self, file_path: Path, total: int = None):\n", - " \"\"\"\n", - " filepath: Path,\n", - " path to a textual file\n", - " total: int\n", - " if you set total to an integer, we'll have a fixed length interation object\n", - " if you set total to None, we'll read the file till it's over\n", - " \"\"\"\n", - " self.file_path = Path(file_path)\n", - " global SingleFileLiner_fp\n", - " SingleFileLiner_fp = open(file_path, \"r\")\n", - " global text_line_num\n", - " text_line_num = 0\n", - " self.total = total\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"SingleFileLiner:\\t{self.file_path}\"\n", - "\n", - " def __next__(self) -> str:\n", - " global SingleFileLiner_fp\n", - " line = SingleFileLiner_fp.readline()\n", - " global text_line_num\n", - " text_line_num += 1\n", - " if line == \"\":\n", - " if self.total is None:\n", - " raise StopIteration(f\"SingleFileLiner file read finish\")\n", - " logging.warning(f\"looping on {self.file_path}\")\n", - "\n", - " self.restart()\n", - " return line.strip()\n", - "\n", - " def __len__(self):\n", - " if self.total is not None:\n", - " return self.total\n", - " else:\n", - " raise RuntimeError(f\"You have to set total for len(self)\")\n", - "\n", - " def __iter__(self) -> str:\n", - " if self.total is None:\n", - " while True:\n", - " try:\n", - " yield next(self)\n", - " except StopIteration:\n", - " # end the file reading\n", - " break\n", - " else:\n", - " for i in range(self.total):\n", - " yield next(self)\n", - "\n", - " def restart(self) -> None:\n", - " \"\"\"resetart file reader\"\"\"\n", - " global SingleFileLiner_fp\n", - " SingleFileLiner_fp.close()\n", - " SingleFileLiner_fp = open(self.file_path, \"r\")\n", - "\n", - " def split_train_test(\n", - " self,\n", - " val_ratio: float = 0.1,\n", - " remove_original: bool = False,\n", - " logging_interval: int = 1000000) -> Tuple[Path]:\n", - " \"\"\"\n", - " Split the text file into train and test\n", - " :param val_ratio:\n", - " :return:\n", - " \"\"\"\n", - " suffix = self.file_path.suffix\n", - "\n", - " train_file_path = self.file_path.parent / \\\n", - " f\"{self.file_path.stem}_train{suffix}\"\n", - " valid_file_path = self.file_path.parent / \\\n", - " f\"{self.file_path.stem}_valid{suffix}\"\n", - "\n", - " # delete the file if exists\n", - " if remove_original:\n", - " train_file_path.unlink()\n", - " valid_file_path.unlink()\n", - "\n", - " # read and write by line\n", - " with open(self.file_path, \"r\") as f, open(\n", - " train_file_path, \"w\") as f_train, open(\n", - " valid_file_path, \"w\") as f_valid:\n", - " for i, line in enumerate(f):\n", - " if i % 1000000 == 0:\n", - " logging.info(f\"{i}\\tlines processed\")\n", - " if random.random() < val_ratio:\n", - " f_valid.write(line)\n", - " else:\n", - " f_train.write(line)\n", - " return train_file_path, valid_file_path" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [], - "source": [ - "sfl = SingleFileLiner(\"../README.md\")\n", - "from joblib import Parallel, delayed\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "def get_line(x):\n", - "# sleep(1)\n", - " return x\n", - "\n", - "res = Parallel(backend=\"multiprocessing\", n_jobs=6)(delayed(get_line)(i) for i in sfl)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read dataframe row by row" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class DataFrameRowling:\n", - " \"\"\"\n", - " Read dataframe row by row\n", - " \"\"\"\n", - " def __init__(self, df):\n", - " self.df = df\n", - " self.restart()\n", - " \n", - " def restart(self):\n", - " global DataFrameRowling_df\n", - " DataFrameRowling_df = self.df\n", - " global DataFrameRowling_ct\n", - " DataFrameRowling_ct = 0\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"DataFrameRowling:\\t{len(self)} Rows\"\n", - " \n", - " def __len__(self):\n", - " return len(self.df)\n", - " \n", - " def __iter__(self):\n", - " for i in range(len(self)):\n", - " yield next(self)\n", - " \n", - " def __getitem__(self, idx):\n", - " global DataFrameRowling_df\n", - " return DataFrameRowling_df[list(DataFrameRowling_df.index)[idx]]\n", - " \n", - " def __next__(self):\n", - " global DataFrameRowling_df\n", - " global DataFrameRowling_ct\n", - " row = DataFrameRowling_df.loc[list(DataFrameRowling_df.index)[DataFrameRowling_ct]]\n", - " DataFrameRowling_ct+=1\n", - " return row" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "df = pd.DataFrame({\"col1\":list(range(100))})" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "res = Parallel(backend=\"multiprocessing\", n_jobs=6)(delayed(get_line)(i) for i in DataFrameRowling(df))" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "col1 5\n", - "Name: 5, dtype: int64" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res[5]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/10_loop.ipynb b/nbs/10_loop.ipynb deleted file mode 100644 index 5f9c839..0000000 --- a/nbs/10_loop.ipynb +++ /dev/null @@ -1,1921 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Controllable loop process\n", - "> iteration with callbacks" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp loop" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - "import numpy as np\n", - "import pandas as pd\n", - "import math" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class Stuff:\n", - " _is_stuff = True\n", - " def __init__(self,name):\n", - " self.name = name\n", - " self.cases = dict()\n", - " self.funcs = dict()\n", - " \n", - " def __setitem__(self,k,v):\n", - " setattr(self,k,v)\n", - " \n", - " def __setattr__(self,k,v):\n", - " if k not in [\"name\",\"cases\",\"funcs\"]:\n", - " self.cases[k] = v\n", - " else:\n", - " super().__setattr__(k,v)\n", - " \n", - " def __getitem__(self,k):\n", - " return self.cases[k]\n", - " \n", - " def append(self,obj):\n", - " self[f\"{self.name}_{len(self+1)}\"] = obj\n", - " \n", - " def clear(self):\n", - " self.cases = dict()\n", - " \n", - " def __getattr__(self,k):\n", - " if k in self.cases:\n", - " return self.cases[k]\n", - " elif k in self.funcs:\n", - " return self.funcs[k]\n", - " else:\n", - " return super().__getattr__()\n", - " \n", - " def __repr__(self,):\n", - " return f\"🍄:{self.name}[\\n\\t\"+\"\\n\\t\".join(self.cases.keys())+\"\\n]\"\n", - " \n", - " def __len__(self):\n", - " return len(self.cases)\n", - " \n", - " def apply(self,func,scope = None):\n", - " \"\"\"\n", - " func, the function to apply on each element\n", - " scope, list, name the scope of element\n", - " \"\"\"\n", - " def apply_func(*args,**kwargs):\n", - " for k,case in self.cases.items():\n", - " if scope!=None:\n", - " if k not in scope:\n", - " continue\n", - " if type(func)==str:\n", - " self[k] = getattr(case,func)(*args,**kwargs)\n", - " else:\n", - " self[k] = func(case,*args,**kwargs)\n", - " return apply_func\n", - " \n", - " def update(self,dictionary):\n", - " for k, v in dictionary.items():\n", - " self[k]=v\n", - " \n", - " def __call__(self,func=None):\n", - " \"\"\"\n", - " func: str or callable, \n", - " if str, it will run on each subject case's such named function\n", - " if callable, it will call on the callable, with the 1st arg being the subject case\n", - " args: extra args\n", - " kwargs: extra key word arguments\n", - " \n", - " return: a list of returned results\n", - " \"\"\"\n", - " def run_funcs(*args,**kwargs):\n", - " results = dict()\n", - " for k,case in self.cases.items():\n", - " if type(func)==str:\n", - " results.update({k:getattr(case,func)(*args,**kwargs)})\n", - " elif func==None:\n", - " results.update({k:getattr(case,\"__call__\")(*args,**kwargs)})\n", - " else:\n", - " results.update({k:func(case,*args,**kwargs)})\n", - " return results\n", - " return run_funcs" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "words = Stuff(\"model\")\n", - "words[\"G\"] = \"generative_model\"\n", - "words[\"D\"] = \"discriminative_model\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def to_device(single_model,device):\n", - " print(f\"{single_model} put to device:{device}\")\n", - " # single_model.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "generative_model put to device:2\n", - "discriminative_model put to device:2\n" - ] - }, - { - "data": { - "text/plain": [ - "{'G': None, 'D': None}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "words(to_device)(2)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'G': 'GENERATIVE_MODEL', 'D': 'DISCRIMINATIVE_MODEL'}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "words(\"upper\")()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'G': 16, 'D': 20}" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "words(\"__len__\")()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Both of the following syntax works" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'generative_model'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "words.G" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In real cases we can do\n", - "```python\n", - "model(\"to\")(\"cuda:1\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "generative_model put to device:cuda:1\n", - "discriminative_model put to device:cuda:1\n" - ] - }, - { - "data": { - "text/plain": [ - "{'G': None, 'D': None}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "words(to_device)(\"cuda:1\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from tqdm import tqdm\n", - "from types import MethodType\n", - "import inspect\n", - "from functools import partial\n", - "\n", - "def method4all(f):\n", - " \"\"\"\n", - " Use this function as a decorator,\n", - " The decorated function under Loop class can be used on outer layer\n", - " \"\"\"\n", - " setattr(f,\"forall\",True)\n", - " return f\n", - "\n", - "class StorageCore:\n", - " def __init__(self,start_layer):\n", - " self.layers = []\n", - " self.lmap = dict()\n", - " self.forall_pool = dict()\n", - " self.new_layer(start_layer)\n", - " self.i = -1\n", - " self.epoch = -1\n", - " \n", - " def new_layer(self,layer):\n", - " layer.core = self\n", - " self.layers.append(layer)\n", - "\n", - " self.lmap[layer._loop_layer]=layer\n", - " if hasattr(self,layer.name):\n", - " raise KeyError(f\"Layer name already exist: {layer.name}\")\n", - " setattr(self,layer.name,layer)\n", - " self.update_forall(layer)\n", - " self.assign_forall(layer)\n", - " \n", - " def __repr__(self):\n", - " return str(self.lmap)\n", - " \n", - " def for_all_functions(self,obj):\n", - " return dict(inspect.getmembers(obj, \n", - " predicate=lambda x:hasattr(x,\"forall\")))\n", - " \n", - " def update_forall(self,obj):\n", - " self.forall_pool.update(self.for_all_functions(obj))\n", - " \n", - " def assign_forall(self,obj):\n", - " for name,f in self.forall_pool.items():\n", - " setattr(obj,name,f)\n", - "\n", - "class Loop(list):\n", - " \"\"\"\n", - " Basic loop class\n", - " \"\"\"\n", - " _loop_layer = 0.\n", - " _loop_obj = True\n", - " def __init__(self,iterable = [],name=None):\n", - " self.iterable = iterable\n", - " self.next_layer = None\n", - " self.name = name if name!=None else self.__class__.__name__\n", - " \n", - " if hasattr(iterable,\"_loop_layer\"):\n", - " self._loop_layer = iterable._loop_layer + 1\n", - " iterable.next_layer = self\n", - " iterable.core.new_layer(self)\n", - " else:\n", - " self.core = StorageCore(self)\n", - " \n", - " def __call__(self):\n", - " pass\n", - " \n", - " def __repr__(self):\n", - " return f\"layer🍰{self.name}\"\n", - " \n", - " def on(self,iterable):\n", - " self.iterable = iterable\n", - " self._loop_layer = iterable._loop_layer + 1\n", - " \n", - " def func_detail(self,f):\n", - " detail = dict({\"🐍func_name\":f.__name__,\n", - " \"⛰doc\":f.__doc__,\n", - " \"😝var\":\",\".join(f.__code__.co_varnames),\n", - " \"😜names\":\",\".join(f.__code__.co_names),\n", - " })\n", - " return detail\n", - " \n", - " def func_detail_df(self):\n", - " funcs = []\n", - " result = []\n", - " for idx,layer in self.core.lmap.items():\n", - " for fname,f in self.core.for_all_functions(layer).items():\n", - " if id(f) not in funcs:\n", - " dt = self.func_detail(f)\n", - " dt.update(layer=str(layer))\n", - " result.append(dt)\n", - " funcs.append(id(f))\n", - " return pd.DataFrame(result)\n", - " \n", - " def search(self):\n", - " from forgebox.widgets import search_box\n", - " pd.set_option('display.max_colwidth', 0)\n", - " doc_df = self.func_detail_df()\n", - " doc_df[\"⛰doc\"]= doc_df[\"⛰doc\"].apply(lambda x:str(x).replace(\"\\n\",\" \"))\n", - " search_box(doc_df, [\"🐍func_name\",\"⛰doc\",\"layer\",\"😝var\",\"😜names\"])\n", - " \n", - " def summary(self):\n", - " rt = f\"Loop layer {self.name} summary:\\n\"\n", - " rt+= \"=\"*50+\"\\n\"\n", - " funcs = []\n", - " for idx,layer in self.core.lmap.items():\n", - " rt+= f\"🍰layer{idx}\\t{str(layer)}\\n\"\n", - " for fname,f in self.core.for_all_functions(layer).items():\n", - " if id(f) not in funcs:\n", - " rt+=\"\\t\"\n", - " rt+=\"\\n\\t\".join(f\"[{k}]\\t{v}\" for k,v in self.func_detail(f).items())\n", - " rt+=\"\\n\\t\"+\".\"*35+\"\\n\"\n", - " funcs.append(id(f))\n", - " rt+=\"-\"*50+\"\\n\"\n", - " rt+= \"=\"*50+\"\\n\"\n", - " print(rt)\n", - " \n", - " def __len__(self):\n", - " return self.iterable.__len__()\n", - " \n", - " def run(self,run_nb=1):\n", - " \"\"\"\n", - " Run through iteration\n", - " run start_call for every layer on the start of iteration\n", - " run __call__ for every layer for each iteration\n", - " run end_call for every layer on the end of iteration\n", - " \"\"\"\n", - " self.before_1st_callon()\n", - " for i in range(run_nb):\n", - " self.one_epoch()\n", - " self.after_last_callon()\n", - " \n", - " def one_epoch(self):\n", - " first = self.layers[0]\n", - " self.refresh_i()\n", - " first.start_callon()\n", - " for element in first:\n", - " first()\n", - " first.end_callon()\n", - " \n", - " def __getitem__(self,idx):\n", - " return self.iterable[idx]\n", - " \n", - " def __setitem__(self,idx,v):\n", - " self.iterable[idx] = v\n", - " \n", - " def refresh_i(self):\n", - " self.core.i=-1\n", - " self.core.epoch+=1\n", - " \n", - " def update_i(self):\n", - " self.core.i+=1\n", - "\n", - " def downstream_wrap(self,f):return f()\n", - " def downstream_start_wrap(self,f):return f()\n", - " def downstream_end_wrap(self,f):return f()\n", - " def downstream_before_1st_wrap(self,f): return f()\n", - " def downstream_after_last_wrap(self,f): return f()\n", - " \n", - " def downstream(self,f):\n", - " \"\"\"\n", - " set downstream wrapper on callback, \n", - " The callback will happend in the local realm of the deocrated function\n", - " example\n", - " @loop.downstream\n", - " def control_layer(self,callback):\n", - " try:\n", - " callback()\n", - " except:\n", - " print(\"error happened\")\n", - " \"\"\"\n", - " setattr(self,\"downstream_wrap\",MethodType(f,self))\n", - " return f\n", - " \n", - " def downstream_start(self,f):\n", - " \"\"\"\n", - " set downstream wrapper on start callback, \n", - " The start_callback will happend in the local realm of the deocrated function\n", - " example\n", - " @loop.downstream_start\n", - " def control_layer(self,start_callback):\n", - " try:\n", - " start_callback()\n", - " except:\n", - " print(\"error happened\")\n", - " \"\"\"\n", - " setattr(self,\"downstream_start_wrap\",MethodType(f,self))\n", - " return f\n", - " \n", - " def downstream_end(self,f):\n", - " \"\"\"\n", - " set downstream wrapper on end callback, \n", - " The end_callback will happend in the local realm of the deocrated function\n", - " @loop.downstream_end\n", - " def control_layer(self,end_callback):\n", - " try:\n", - " end_callback()\n", - " except:\n", - " print(\"error happened\")\n", - " \"\"\"\n", - " setattr(self,\"downstream_end_wrap\",MethodType(f,self))\n", - " return f\n", - " \n", - " def callon(self):\n", - " self()\n", - " self.downstream_wrap(self.iter_cb)\n", - " \n", - " def start_callon(self):\n", - " self.start_call()\n", - " self.downstream_start_wrap(self.start_cb)\n", - " \n", - " def end_callon(self):\n", - " self.end_call()\n", - " self.downstream_end_wrap(self.end_cb)\n", - " \n", - " def before_1st_callon(self):\n", - " self.before_1st_call()\n", - " self.downstream_before_1st_wrap(self.before_1st_cb)\n", - " \n", - " def after_last_callon(self):\n", - " self.after_last_call()\n", - " self.downstream_before_1st_wrap(self.after_last_cb)\n", - " \n", - " def iter_cb(self):\n", - " \"\"\"\n", - " call back during each iteration\n", - " \"\"\"\n", - " if self.next_layer!=None:\n", - " self.next_layer.callon()\n", - " self.after()\n", - " \n", - " def after(self):\n", - " pass\n", - " \n", - " def start_cb(self):\n", - " \"\"\"\n", - " callback at the start of iteration\n", - " \"\"\"\n", - " if self.next_layer!=None:\n", - " self.next_layer.start_callon()\n", - " \n", - " def end_cb(self):\n", - " \"\"\"\n", - " callback at the end of iteration\n", - " \"\"\"\n", - " if self.next_layer!=None:\n", - " self.next_layer.end_callon()\n", - " \n", - " def before_1st_cb(self):\n", - " \"\"\"\n", - " callback before the 1st of iteration\n", - " \"\"\"\n", - " if self.next_layer!=None:\n", - " self.next_layer.before_1st_callon()\n", - " \n", - " def after_last_cb(self):\n", - " \"\"\"\n", - " callback after the last of iteration\n", - " \"\"\"\n", - " if self.next_layer!=None:\n", - " self.next_layer.after_last_callon()\n", - " \n", - " def before_1st_call(self):pass\n", - " def after_last_call(self):pass\n", - " def start_call(self):pass\n", - " def end_call(self):pass\n", - " \n", - " def __iter__(self,):\n", - " for element in self.iterable:\n", - " if self._loop_layer ==0:\n", - " self.update_i()\n", - " self.core.element = element\n", - " self.callon()\n", - " yield self.element\n", - " \n", - " def __getattr__(self,k):\n", - " return getattr(self.core,k)\n", - " \n", - " def is_newloop(self):\n", - " \"\"\"\n", - " return Bool:Is this a new loop ready to start\n", - " \"\"\"\n", - " return (self.i==-1 or self.i==self.__len__()-1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loop-in-loop concept\n", - "Loop and its inheriting descendants runs on iterable, incuding other ```Loop```" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "layer🍰hahahha" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "l = Loop(range(5,10),name = \"hahahha\")\n", - "l" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "class Printer(Loop):\n", - " def __call__(self):\n", - " print(f\"i_{self.core.i}\",end=\"\\t\")\n", - " print(self.core.element)\n", - " \n", - " def start_call(self):\n", - " print(f\"start of epoch {self.epoch}\")\n", - " \n", - " def end_call(self):\n", - " print(f\"end of epoch {self.epoch}\")\n", - "l2 = Printer(l)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "start of epoch 0\n", - "i_0\t5\n", - "i_1\t6\n", - "i_2\t7\n", - "i_3\t8\n", - "i_4\t9\n", - "end of epoch 0\n" - ] - } - ], - "source": [ - "l2.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Progress bar" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class ProgressBar(Loop):\n", - " def __init__(self,iterable=[],jupyter = True,mininterval = 1.,leave=False):\n", - " super().__init__(iterable,\"ProgressBar\")\n", - " \"\"\"\n", - " jupyter:bool, using the jupyter progressbar (or the console pb)?\n", - " default True\n", - " mininterval: minimun update interval\n", - " leave:bool, if true, we do not remove the progress bar at the end\n", - " default: False\n", - " \"\"\"\n", - " \n", - " if jupyter: # jupyter widget\n", - " from tqdm.notebook import tqdm\n", - " \n", - " else: # compatible for console print\n", - " from tqdm import tqdm\n", - " \n", - " self.tqdm = tqdm\n", - " self.leave = leave\n", - " self.mininterval = mininterval\n", - " self.data = dict()\n", - " \n", - " @method4all\n", - " def pgbar_data(self,data):\n", - " \"\"\"\n", - " update progress bar with python dictionary\n", - " data: python dictionary\n", - " \"\"\"\n", - " self.t.set_postfix(data)\n", - " \n", - " @method4all\n", - " def pgbar_description(self,text):\n", - " \"\"\"\n", - " update progress bar prefix with text string\n", - " \"\"\"\n", - " self.t.set_description_str(f\"{text}\")\n", - " \n", - " def start_call(self):\n", - " self.create_bar()\n", - " \n", - " def end_call(self):\n", - " self.t.close()\n", - " \n", - " def __call__(self):\n", - " self.t.update(1)\n", - " \n", - " def create_bar(self):\n", - " self.t = self.tqdm(total=len(self.iterable),leave=self.leave,\n", - " mininterval=self.mininterval)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "from tqdm.notebook import tqdm,tqdm_notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b3b38d446a1e4d67b5204f94cc6ed81d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "pb = ProgressBar(Loop(range(5,10)),jupyter = True,leave=True)\n", - "pb.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Downstream wraper\n", - "> A unique way to contain downstream task in a function, hence a good Error Handling mechanism" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In cases you don't want error stop the iteration, meanwhile taking down all the error message." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def error_tolerate(self,downstream_func):\n", - " \"\"\"\n", - " downstream_func:Downstream function\n", - " \"\"\"\n", - " try:\n", - " downstream_func()\n", - " except Exception as e:\n", - " self.errors.append(dict(stage=downstream_func.__name__,\n", - " i=self.i,\n", - " epoch=self.epoch,\n", - " error=e))\n", - " \n", - "class Tolerate(Loop):\n", - " \"\"\"\n", - " Tolerate any error happened downstream\n", - " layer2 = Tolerate(upstream_iterable)\n", - " # build downstream tasks\n", - " layer3 = OtherApplication(layer2)\n", - " layer3.run()\n", - " # show the happened error message\n", - " layer3.error_list()\n", - " \"\"\"\n", - " def __init__(self,iterable = []):\n", - " super().__init__(iterable,)\n", - " self.errors = list()\n", - " # wrap downstream task with error tolerate\n", - " for decorator in [self.downstream,\n", - " self.downstream_start,\n", - " self.downstream_end]:\n", - " decorator(error_tolerate)\n", - " \n", - " @method4all\n", - " def error_list(self):\n", - " \"\"\"\n", - " A list of errors happend so far\n", - " \"\"\"\n", - " return self.errors\n", - " \n", - " def end_call(self,):\n", - " le = len(self.error_list())\n", - " if le>0:\n", - " print(f\"WARNING:{le} errors\")\n", - "\n", - "class LambdaCall(Loop):\n", - " def __init__(self,iterable = [],\n", - " func = lambda x:x,\n", - " start_func = lambda x:x,\n", - " end_func = lambda x:x\n", - " ):\n", - " super().__init__(iterable,name=f\"Lambda<{hex(id(self))}>\")\n", - " self.func = func\n", - " self.start_func = end_func\n", - " self.end_func = end_func\n", - " \n", - " def __call__(self):\n", - " self.func(self)\n", - " \n", - " def start_call(self):\n", - " self.start_func(self)\n", - " \n", - " def end_call(self):\n", - " self.end_func(self)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "l1 = ProgressBar(Loop(range(1,10)))\n", - "error_loop = LambdaCall(l1,func= lambda x:(5-x.element)**(-2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Running this will yield error" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - "error_loop.run()\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "WARNING:5 errors\n" - ] - } - ], - "source": [ - "def error_on_purpose(x):\n", - " if x.element>5:\n", - " raise ValueError(f\"value too small, {x.element}\")\n", - "\n", - "l1 = ProgressBar(Loop(range(1,10)))\n", - "# add the tolerate layer in between\n", - "l2 = Tolerate(l1)\n", - "# tolerate can only tolerate the erorr on downstream\n", - "tolerated_loop = LambdaCall(l2,func=error_on_purpose,end_func=error_on_purpose)\n", - "l2.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "you can use ```error_list``` function to print all the happened error" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
stageiepocherror
0start_cb-10'StorageCore' object has no attribute 'element'
1iter_cb50value too small, 6
2iter_cb60value too small, 7
3iter_cb70value too small, 8
4iter_cb80value too small, 9
5end_cb80value too small, 9
\n", - "
" - ], - "text/plain": [ - " stage i epoch error\n", - "0 start_cb -1 0 'StorageCore' object has no attribute 'element'\n", - "1 iter_cb 5 0 value too small, 6\n", - "2 iter_cb 6 0 value too small, 7\n", - "3 iter_cb 7 0 value too small, 8\n", - "4 iter_cb 8 0 value too small, 9\n", - "5 end_cb 8 0 value too small, 9" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "pd.DataFrame(l2.error_list())" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loop layer Tolerate summary:\n", - "==================================================\n", - "🍰layer0.0\tlayer🍰Loop\n", - "--------------------------------------------------\n", - "🍰layer1.0\tlayer🍰ProgressBar\n", - "\t[🐍func_name]\tpgbar_data\n", - "\t[⛰doc]\t\n", - " update progress bar with python dictionary\n", - " data: python dictionary\n", - " \n", - "\t[😝var]\tself,data\n", - "\t[😜names]\tt,set_postfix\n", - "\t...................................\n", - "\t[🐍func_name]\tpgbar_description\n", - "\t[⛰doc]\t\n", - " update progress bar prefix with text string\n", - " \n", - "\t[😝var]\tself,text\n", - "\t[😜names]\tt,set_description_str\n", - "\t...................................\n", - "--------------------------------------------------\n", - "🍰layer2.0\tlayer🍰Tolerate\n", - "\t[🐍func_name]\terror_list\n", - "\t[⛰doc]\t\n", - " A list of errors happend so far\n", - " \n", - "\t[😝var]\tself\n", - "\t[😜names]\terrors\n", - "\t...................................\n", - "--------------------------------------------------\n", - "🍰layer3.0\tlayer🍰Lambda<0x119422e30>\n", - "--------------------------------------------------\n", - "==================================================\n", - "\n" - ] - } - ], - "source": [ - "l2.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can search for the function documentation" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9426613e40fa44589f739cf71688301c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(Text(value='', description='KeyWord'), Output()), _dom_classes=('widget-interact',))" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "l2.search()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Application upon progress bar\n", - "The function ```pgbar_data``` is decorated under ```method4all```, hence the function can be used else where" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "16843ddafaa74ad4bc68834313673457", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from datetime import datetime\n", - "class Application(Loop):\n", - " def __call__(self):\n", - " # create some data\n", - " loop_data = dict(epoch=self.epoch,i = self.i,\n", - " val = self.element,)\n", - " # update data to progress bar\n", - " self.pgbar_data(loop_data)\n", - " self.pgbar_description(str(datetime.now().strftime(\"%H:%M:%S\")))\n", - "\n", - "l3 = Application(ProgressBar(Loop(range(5,10)),jupyter=True,leave=True))\n", - "l3.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Print out the summary of the loop" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loop layer Application summary:\n", - "==================================================\n", - "🍰layer0.0\tlayer🍰Loop\n", - "--------------------------------------------------\n", - "🍰layer1.0\tlayer🍰ProgressBar\n", - "\t[🐍func_name]\tpgbar_data\n", - "\t[⛰doc]\t\n", - " update progress bar with python dictionary\n", - " data: python dictionary\n", - " \n", - "\t[😝var]\tself,data\n", - "\t[😜names]\tt,set_postfix\n", - "\t...................................\n", - "\t[🐍func_name]\tpgbar_description\n", - "\t[⛰doc]\t\n", - " update progress bar prefix with text string\n", - " \n", - "\t[😝var]\tself,text\n", - "\t[😜names]\tt,set_description_str\n", - "\t...................................\n", - "--------------------------------------------------\n", - "🍰layer2.0\tlayer🍰Application\n", - "--------------------------------------------------\n", - "==================================================\n", - "\n" - ] - } - ], - "source": [ - "l3.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1d0d9ea301c44ec2981dedbe97517553", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "l3.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Event\n", - "\n", - "Event allows add-hoc callbacks to be created" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - " \n", - "class Event(Loop):\n", - " \"\"\"\n", - " An event is the landmark with in 1 iteration\n", - " \"\"\"\n", - " def __init__(self,iterable=[]):\n", - " event_name = self.__class__.__name__\n", - " super().__init__(iterable,event_name)\n", - " self.event_name = event_name\n", - " self.cbs = []\n", - " self.create_cb_deco(\"on\")\n", - " self.create_cb_deco(\"before_1st\")\n", - " self.create_cb_deco(\"every_start\")\n", - " self.create_cb_deco(\"every_end\")\n", - " self.create_cb_deco(\"after_last\")\n", - " self.core.update_forall(self)\n", - " self.core.assign_forall(self)\n", - " \n", - " def __repr__(self):\n", - " return f\"event🌏{self.event_name}\"\n", - " \n", - " def create_cb_deco(self,moment):\n", - " event_name = self.event_name\n", - " def wraper(cls,f):return getattr(self,moment)(f)\n", - " wraper.__name__ = f\"{moment}_{event_name}\"\n", - " wraper.__doc__ = f\"\"\"\n", - " Append new {moment} callback for event:{event_name}\n", - " Use this function as decorator\n", - " \"\"\"\n", - " setattr(self,wraper.__name__,MethodType(method4all(wraper),self))\n", - " \n", - " def __call__(self):\n", - " if len(self.cbs)>0:\n", - " self.cbs[0].callon()\n", - " \n", - " def start_call(self):\n", - " if len(self.cbs)>0:\n", - " self.cbs[0].start_callon()\n", - " \n", - " def end_call(self):\n", - " if len(self.cbs)>0:\n", - " self.cbs[0].end_callon()\n", - " \n", - " def on(self,f):\n", - " class EventCB(Loop):\n", - " def __init__(self_,iterable=[]):\n", - " super().__init__(iterable=iterable,\n", - " name = f\"ON_{self.event_name}_{self.cbs.__len__()+1}_{f.__name__}\")\n", - " self_.f = f\n", - " \n", - " def __call__(self_): self_.f(self)\n", - " \n", - " new_cb = EventCB()\n", - " self.new_event_cb(new_cb)\n", - " return f\n", - "\n", - " def every_start(self,f):\n", - " class EventCbEveryStart(Loop):\n", - " def __init__(self_,iterable=[]):\n", - " super().__init__(iterable=iterable,\n", - " name = f\"EVERY_START_{self.event_name}_{self.cbs.__len__()+1}_{f.__name__}\")\n", - " self_.f = f\n", - " \n", - " def start_call(self_): self_.f(self)\n", - " \n", - " new_cb = EventCbEveryStart()\n", - " self.new_event_cb(new_cb)\n", - " return f\n", - " \n", - " def before_1st(self,f):\n", - " class EventCbBefore1st(Loop):\n", - " def __init__(self_,iterable=[]):\n", - " super().__init__(iterable=iterable,\n", - " name = f\"BEFORE_1ST_{self.event_name}_{self.cbs.__len__()+1}_{f.__name__}\")\n", - " self_.f = f\n", - " \n", - " def before_1st_call(self_): self_.f(self)\n", - " \n", - " new_cb = EventCbBefore1st()\n", - " self.new_event_cb(new_cb)\n", - " return f\n", - " \n", - " def every_end(self,f):\n", - " class EventCbEveryEnd(Loop):\n", - " def __init__(self_,iterable=[]):\n", - " super().__init__(iterable=iterable,\n", - " name = f\"EVERY_END_{self.event_name}_{self.cbs.__len__()+1}_{f.__name__}\")\n", - " self_.f = f\n", - " \n", - " def end_call(self_): self_.f(self)\n", - " \n", - " new_cb = EventCbEveryEnd()\n", - " self.new_event_cb(new_cb)\n", - " return f\n", - " \n", - " def after_last(self,f):\n", - " class EventCbAfter(Loop):\n", - " def __init__(self_,iterable=[]):\n", - " super().__init__(iterable=iterable,\n", - " name = f\"AFTER_LAST_{self.event_name}_{self.cbs.__len__()+1}_{f.__name__}\")\n", - " self_.f = f\n", - " \n", - " def after_last_call(self_): self_.f(self)\n", - " \n", - " new_cb = EventCbAfter()\n", - " self.new_event_cb(new_cb)\n", - " return f\n", - " \n", - " def new_event_cb(self,new_cb):\n", - " new_cb.core.update_forall(new_cb)\n", - " new_cb.core.assign_forall(new_cb)\n", - " new_cb.core = self.core\n", - " if len(self.cbs)>0:\n", - " self.cbs[-1].next_layer = new_cb\n", - "# self.cbs[-1].new_layer(new_cb)\n", - " self.cbs.append(new_cb)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "class ToString(Loop):\n", - " def __call__(self,):\n", - " self.core.element = f\"variable_{self.i}\"\n", - "\n", - "class ChangeString(Event):\n", - " def __init__(self,iterable=[]):\n", - " super().__init__(iterable=iterable)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def get_frame(li):\n", - " layer1 = ProgressBar(Loop(li,\"base\"),leave=True)\n", - " layer2 = ToString(layer1,)\n", - " layer3 = ChangeString(layer2)\n", - " # you can use this \"on\" syntax\n", - " @layer3.ChangeString.on\n", - " def trim(self):\n", - " self.core.element = f\"Var_{self.element}\"\n", - " \n", - " # or this syntax\n", - " @layer3.on_ChangeString\n", - " def update(self):\n", - " self.pgbar_description(self.element)\n", - " \n", - " @layer3.before_1st_ChangeString\n", - " def print_start(self):\n", - " \"\"\"\n", - " decorated\n", - " \"\"\"\n", - " print(f\"start_{self.epoch}_{self.i}\")\n", - " print(self.name)\n", - " return layer3" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "l = get_frame(range(10))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{0.0: layer🍰base, 1.0: layer🍰ProgressBar, 2.0: layer🍰ToString, 3.0: event🌏ChangeString}" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "l.core" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[layer🍰ON_ChangeString_1_trim,\n", - " layer🍰ON_ChangeString_2_update,\n", - " layer🍰BEFORE_1ST_ChangeString_3_print_start]" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "l.cbs" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loop layer ChangeString summary:\n", - "==================================================\n", - "🍰layer0.0\tlayer🍰base\n", - "--------------------------------------------------\n", - "🍰layer1.0\tlayer🍰ProgressBar\n", - "\t[🐍func_name]\tpgbar_data\n", - "\t[⛰doc]\t\n", - " update progress bar with python dictionary\n", - " data: python dictionary\n", - " \n", - "\t[😝var]\tself,data\n", - "\t[😜names]\tt,set_postfix\n", - "\t...................................\n", - "\t[🐍func_name]\tpgbar_description\n", - "\t[⛰doc]\t\n", - " update progress bar prefix with text string\n", - " \n", - "\t[😝var]\tself,text\n", - "\t[😜names]\tt,set_description_str\n", - "\t...................................\n", - "--------------------------------------------------\n", - "🍰layer2.0\tlayer🍰ToString\n", - "--------------------------------------------------\n", - "🍰layer3.0\tevent🌏ChangeString\n", - "\t[🐍func_name]\tafter_last_ChangeString\n", - "\t[⛰doc]\t\n", - " Append new after_last callback for event:ChangeString\n", - " Use this function as decorator\n", - " \n", - "\t[😝var]\tcls,f\n", - "\t[😜names]\tgetattr\n", - "\t...................................\n", - "\t[🐍func_name]\tbefore_1st_ChangeString\n", - "\t[⛰doc]\t\n", - " Append new before_1st callback for event:ChangeString\n", - " Use this function as decorator\n", - " \n", - "\t[😝var]\tcls,f\n", - "\t[😜names]\tgetattr\n", - "\t...................................\n", - "\t[🐍func_name]\tevery_end_ChangeString\n", - "\t[⛰doc]\t\n", - " Append new every_end callback for event:ChangeString\n", - " Use this function as decorator\n", - " \n", - "\t[😝var]\tcls,f\n", - "\t[😜names]\tgetattr\n", - "\t...................................\n", - "\t[🐍func_name]\tevery_start_ChangeString\n", - "\t[⛰doc]\t\n", - " Append new every_start callback for event:ChangeString\n", - " Use this function as decorator\n", - " \n", - "\t[😝var]\tcls,f\n", - "\t[😜names]\tgetattr\n", - "\t...................................\n", - "\t[🐍func_name]\ton_ChangeString\n", - "\t[⛰doc]\t\n", - " Append new on callback for event:ChangeString\n", - " Use this function as decorator\n", - " \n", - "\t[😝var]\tcls,f\n", - "\t[😜names]\tgetattr\n", - "\t...................................\n", - "--------------------------------------------------\n", - "==================================================\n", - "\n" - ] - } - ], - "source": [ - "l.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[layer🍰base, layer🍰ProgressBar, layer🍰ToString, event🌏ChangeString]" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "l.layers" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "53165b9cead148bbbe51817d3054ad9b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "l.run()" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "layer🍰base\n", - "None\n", - "None\n", - "None\n" - ] - } - ], - "source": [ - "layer = l.layers[0]\n", - "print(layer)\n", - "while layer.next_layer!=None:\n", - " layer=layer.next_layer\n", - " print(layer.start_call.__doc__)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parallel" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from joblib import Parallel,delayed" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "from math import sqrt" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "def heavy_calc(x):\n", - " sleep(1)\n", - " return sqrt(x)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 933 µs, sys: 1.41 ms, total: 2.35 ms\n", - "Wall time: 10 s\n" - ] - }, - { - "data": { - "text/plain": [ - "[0.0,\n", - " 1.0,\n", - " 1.4142135623730951,\n", - " 1.7320508075688772,\n", - " 2.0,\n", - " 2.23606797749979,\n", - " 2.449489742783178,\n", - " 2.6457513110645907,\n", - " 2.8284271247461903,\n", - " 3.0]" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "list(map(heavy_calc,range(10)))" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 26.5 ms, sys: 21.8 ms, total: 48.3 ms\n", - "Wall time: 2.79 s\n" - ] - }, - { - "data": { - "text/plain": [ - "[0.0,\n", - " 1.0,\n", - " 1.4142135623730951,\n", - " 1.7320508075688772,\n", - " 2.0,\n", - " 2.23606797749979,\n", - " 2.449489742783178,\n", - " 2.6457513110645907,\n", - " 2.8284271247461903,\n", - " 3.0]" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "Parallel(n_jobs=5,)(delayed(heavy_calc)(i) for i in range(10))" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "( at 0x11b3761d0>,)\n", - "0\n", - "1\n", - "2\n", - "3\n", - "4\n" - ] - } - ], - "source": [ - "def showstuff(*args):\n", - " print(args)\n", - " for i in args[0]:\n", - " print(i)\n", - "showstuff(a for a in range(5))" - ] - }, - { - "cell_type": "code", - "execution_count": 123, - "metadata": {}, - "outputs": [], - "source": [ - "class MultiProc:\n", - " def __init__(self,loop,n_jobs = 6):\n", - " self.loop = loop\n", - " self.n_jobs = n_jobs\n", - " self.parallel = Parallel(n_jobs=n_jobs)\n", - " \n", - " def __enter__(self):\n", - " loop = self.loop\n", - " def one_epoch_parallel(loop):\n", - " first = loop.layers[0]\n", - " loop.refresh_i()\n", - " first.start_callon()\n", - " print(1)\n", - " self.parallel(delayed(first.__call__)() for element in first)\n", - " print(2)\n", - " first.end_callon()\n", - " self.old = loop.one_epoch\n", - " setattr(loop,\"one_epoch\",MethodType(one_epoch_parallel,loop))\n", - " return\n", - " \n", - " def __exit__(self,exc_type,exc_val,exc_tb):\n", - " setattr(self.loop,\"one_epoch\",self.old)\n", - " return " - ] - }, - { - "cell_type": "code", - "execution_count": 124, - "metadata": {}, - "outputs": [], - "source": [ - "class Calc(Loop):\n", - " def __init__(self,iterable):\n", - " super().__init__(iterable=iterable)\n", - " \n", - " def __call__(self):\n", - " sleep(1)\n", - " self.pgbar_data(dict(sqrt=sqrt(self.element)))\n", - " \n", - " def end_call(self):\n", - " print(\"done\")" - ] - }, - { - "cell_type": "code", - "execution_count": 125, - "metadata": {}, - "outputs": [], - "source": [ - "basic_loop = ProgressBar(Loop(range(10)))\n", - "test_li = Calc(basic_loop)" - ] - }, - { - "cell_type": "code", - "execution_count": 126, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "done\n", - "CPU times: user 55.5 ms, sys: 8.47 ms, total: 64 ms\n", - "Wall time: 10.1 s\n" - ] - } - ], - "source": [ - "%%time\n", - "test_li.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Chunkify for batch cutting" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def chunkify(*iterables,bs = 32):\n", - " if len(iterables)>1:\n", - " return list(tuple(l[i*bs:(i+1)*bs] for l in iterables) \\\n", - " for i in range(math.ceil(len(iterables[0])/bs)))\n", - " else:\n", - " return list(iterables[0][i*bs:(i+1)*bs] \\\n", - " for i in range(math.ceil(len(iterables[0])/bs)))" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[([0, 1, 2], [10, 11, 12]),\n", - " ([3, 4, 5], [13, 14, 15]),\n", - " ([6, 7, 8], [16, 17, 18]),\n", - " ([9], [19])]" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "chunkify(list(range(10)),list(range(10,20)),bs=3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/12_static_file.ipynb b/nbs/12_static_file.ipynb deleted file mode 100644 index a85468e..0000000 --- a/nbs/12_static_file.ipynb +++ /dev/null @@ -1,166 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Get static file\n", - "> get none-python file, like js and html" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp static_file" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from pathlib import Path \n", - "def get_static()->Path: \n", - " \"\"\"\n", - " return the absolute path of forgebox.static\n", - " \"\"\"\n", - " import forgebox \n", - " return Path(forgebox.__path__[0])/\"static\" \n", - "\n", - "def open_static(relative_path:str)->str:\n", - " file = get_static()/relative_path\n", - " with open(file,\"r\") as f:\n", - " return f.read()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The static file directory" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "PosixPath('/Users/xiaochen.zhang/github/forgebox/forgebox/static')" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_static()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Open relative path" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"
\\n

Masked language modeling visualization

\\n \\n\\n
\\n
\"" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "open_static(\"mlm/visual.html\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "var mlm_visualize = (data, text, output_id) => {\n", - " var {\n", - " Sentence,\n", - " ControlBar,\n", - " att_obj\n", - " } = attention_factory(data, text, output_id)\n", - "\n", - " var sentence = new Sentence();\n", - " var control_bar = new ControlBar({\n", - " nb_heads: att_obj.attention.length,\n", - " current_head: 0\n", - " });\n", - "\n", - " control_bar.sensitive.add_callback(\n", - " control_bar.func_print_change,\n", - " control_bar.change_current_head(att_obj),\n", - " )\n", - "\n", - " $(`#${output_id}`).append(control_bar.dom)\n", - " \n" - ] - } - ], - "source": [ - "print(open_static(\"mlm/visual.js\")[:500])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/15_loopstack.ipynb b/nbs/15_loopstack.ipynb deleted file mode 100644 index 3c6bdb2..0000000 --- a/nbs/15_loopstack.ipynb +++ /dev/null @@ -1,1341 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 15 LoopStack\n", - "> Stack loops in layers" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp loopstack" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from forgebox.loop import StorageCore,Loop,ProgressBar,Tolerate,Event,Stuff,chunkify\n", - "from types import MethodType\n", - "from datetime import datetime\n", - "import numpy as np\n", - "import pandas as pd\n", - "from time import sleep" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To run through iterations quickly, we use the california housing data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((20640, 8), (20640,))" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.datasets.california_housing import fetch_california_housing\n", - "x,y = fetch_california_housing(return_X_y = True)\n", - "\n", - "x.shape, y.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Normalized Data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Preview x input" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "def norm(m,s): return lambda x: (x-m)/s\n", - "def denorm(m,s): return lambda x: (x*s)+m\n", - "\n", - "x_normed = norm(x.mean(0),x.std(0))(x)\n", - "y_normed = norm(y.mean(),y.std())(y)\n", - "\n", - "x_normed = np.clip(x_normed,-2,2)\n", - "y_normed = np.clip(y_normed,-2,2)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([-0.04333 , 0.000478, -0.0397 , -0.04486 , -0.0554 , -0.01229 ,\n", - " -0.007812, 0.001071], dtype=float16),\n", - " array([0.85610596, 0.99901853, 0.53787672, 0.32598025, 0.73420682,\n", - " 0.09438235, 0.98221114, 0.99421595]),\n", - " -0.0302,\n", - " 0.928904447452199)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x_normed.mean(0).astype(np.float16),x_normed.std(0),\\\n", - "y_normed.mean().astype(np.float16),y_normed.std()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMkAAAD4CAYAAABG4MINAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAK6ElEQVR4nO3dbaiX9R3H8c/Hc6PneHOU5ZipTVthRNswzmpNqFEb2A01xh4Y2FgMpLZabYOoEQTbs21EezAaYkUsmQ+sQYywgmq0tYlHC5pZTbTyZKKV96bnqN898ATu7By/l3T9ui7b+wVB5++/r1+Ovc/1v/O6HBECML4JTS8AtB2RAAkiARJEAiSIBEh0lhja3dkbPV19tc+dOH+o9pkH3u+tfaYkHS/ynZW6DpR5NXJ4imuf2TPtcO0zJenYm8eKzN0fu9+PiJmjby/yR9nT1afLzvth7XPPe3RL7TNffPhrtc+UpCMziozVF9YeKTJ3x6UTa5950dVv1D5TkvZ9c2+Ruc8Or3p7rNt5uAUkiARIEAmQIBIgQSRAgkiARKVIbC+2/YbtzbbvLr0U0CZpJLY7JP1e0tWSLpR0o+0LSy8GtEWVI8klkjZHxJaIGJK0StINZdcC2qNKJLMlbTvp68GR2/6L7WW2B2wPDB07VNd+QOOqRDLWh3r+5wNEEbE8Ivojor+7o8znoYAmVIlkUNLck76eI2l7mXWA9qkSyTpJ59ueb7tb0hJJT5ZdC2iP9FPAEXHU9m2SnpbUIenhiNhYfDOgJSp9VD4inpL0VOFdgFbiHXcgQSRAgkiABJEACSIBEkVOBDHU16lt13yu9rnHb+2ufeaMvjInVth7bv27StKuH39UZG7HS/WfCOLo8UI/gzs6yswdHvtmjiRAgkiABJEACSIBEkQCJIgESBAJkCASIEEkQIJIgASRAAkiARJEAiSIBEgQCZAgEiBBJECCSIAEkQAJIgESRAIkipwtpXvfMc15Znftc/dc2Ff7zEkfHq19piQNTx3rsi6f3PQ/Ti0yd+/8ImOLsMt8b8fDkQRIEAmQIBIgQSRAgkiABJEACSIBEmkktufaft72Jtsbbd/xaSwGtEWVNxOPSvp5RGywPVXSetvPRsRrhXcDWiE9kkTEexGxYeTf90vaJGl26cWAtjit5yS250laKGntGL+2zPaA7YGho4fq2Q5ogcqR2J4i6XFJd0bEvtG/HhHLI6I/Ivq7O3vr3BFoVKVIbHfpRCArI+KJsisB7VLl1S1LekjSpoi4v/xKQLtUOZIsknSTpCttvzLyzzWF9wJaI30JOCL+JunT/QA/0CK84w4kiARIEAmQIBIgUeREELPO+0D3/nll7XOXrrml9pnqifpnSpq+rshYTfpgqMjcDy/oqX1m54Tjtc+UpIgyf2bj4UgCJIgESBAJkCASIEEkQIJIgASRAAkiARJEAiSIBEgQCZAgEiBBJECCSIAEkQAJIgESRAIkiARIEAmQIBIgQSRAosjZUt59e6buvXVZ7XMXfPhR7TMPza7/LCGStOsrRcZq58Iy+04Yrn/m7iNlLsHR1VvmLCw6PPbNHEmABJEACSIBEkQCJIgESBAJkCASIHE6l6jusP2y7b+UXAhom9M5ktwhaVOpRYC2qnod9zmSrpW0ouw6QPtUPZI8IOkuSeN+HsD2MtsDtgeGhw7WshzQBmkktq+TtDMi1p/qfhGxPCL6I6K/q3tybQsCTatyJFkk6Xrbb0laJelK248V3QpokTSSiLgnIuZExDxJSyQ9FxFLi28GtATvkwCJ0/r7JBHxgqQXimwCtBRHEiBBJECCSIAEkQAJIgESRc6Wcqzb2n9OV+1zn3+k/o+OXfyrW2ufKUlymbFnv7CnyNxti6fXPnPGxEO1z5SkvXvKfA/Gw5EESBAJkCASIEEkQIJIgASRAAkiARJEAiSIBEgQCZAgEiBBJECCSIAEkQAJIgESRAIkiARIEAmQIBIgQSRAgkiARJGzpUydeVBX3LK29rnXXPHd2mfuuyVqnylJZ20oM3fnpX1F5g5PqX/ffUOTap8pSRN6e4vM1YFxfr8yvxvw2UEkQIJIgASRAAkiARJEAiSIBEhUisT2dNurbb9ue5Pty0ovBrRF1TcTfydpTUR8z3a3pELv5gDtk0Zie5qkyyX9QJIiYkjSUNm1gPao8nDrXEm7JD1i+2XbK2xPHn0n28tsD9ge+Gj3kdoXBZpSJZJOSRdLejAiFko6KOnu0XeKiOUR0R8R/T0zJta8JtCcKpEMShqMiI8/sbhaJ6IB/i+kkUTEDknbbC8YuekqSa8V3Qpokaqvbt0uaeXIK1tbJN1cbiWgXSpFEhGvSOovvAvQSrzjDiSIBEgQCZAgEiBBJECiyNlSDu6YrH/8+pLa57701z/UPvOrv/lR7TMlaf85Rcaqb+vxInOH+ur/eTmt+3DtMyVp78GDReaOhyMJkCASIEEkQIJIgASRAAkiARJEAiSIBEgQCZAgEiBBJECCSIAEkQAJIgESRAIkiARIEAmQIBIgQSRAgkiARJETQXR9/rDm3P7v2udevXhJ7TMPfr/MiRWmbi3z82fBnRuLzP3nmi/XPnPf0KTaZ0pSx7RpReZq79g3cyQBEkQCJIgESBAJkCASIEEkQIJIgESlSGz/1PZG2/+y/SfbZV4AB1oojcT2bEk/kdQfERdJ6pBU/7t6QEtVfbjVKanHdqekXknby60EtEuV67i/K+m3kt6R9J6kvRHxzOj72V5me8D2wJE9Za5LATShysOtGZJukDRf0tmSJtteOvp+EbE8Ivojon/idJ6y4LOjysOtb0naGhG7ImJY0hOSvlF2LaA9qkTyjqSv2+61bUlXSdpUdi2gPao8J1krabWkDZJeHflvlhfeC2iNSn+fJCLuk3Rf4V2AVuIddyBBJECCSIAEkQAJIgESRc6WMny8Q+8e6Kt97t/XrKp95peeu7n2mZI09EFPkblbf3lBkbkd9Z8sRbN699U/VNL2ffuLzB0PRxIgQSRAgkiABJEACSIBEkQCJIgESBAJkCASIEEkQIJIgASRAAkiARJEAiSIBEgQCZAgEiBBJECCSIAEkQAJIgESjoj6h9q7JL1d4a5nSXq/9gXKOZP2PZN2ldqx7xcjYuboG4tEUpXtgYjob2yB03Qm7Xsm7Sq1e18ebgEJIgESTUdypl0M6Eza90zaVWrxvo0+JwHOBE0fSYDWIxIg0VgkthfbfsP2Ztt3N7VHxvZc28/b3mR7o+07mt6pCtsdtl+2/ZemdzkV29Ntr7b9+sj3+LKmdxqtkecktjskvSnp25IGJa2TdGNEvPapL5OwPUvSrIjYYHuqpPWSvtPGXU9m+2eS+iVNi4jrmt5nPLYflfRiRKyw3S2pNyL2NL3XyZo6klwiaXNEbImIIUmrJN3Q0C6nFBHvRcSGkX/frxPXsJ/d7FanZnuOpGslrWh6l1OxPU3S5ZIekqSIGGpbIFJzkcyWtO2krwfV8v/xJMn2PEkLJa1tdpPUA5LuknS86UUS50raJemRkYeGK2xPbnqp0ZqKxGPc1urXom1PkfS4pDsjoswlnGpg+zpJOyNifdO7VNAp6WJJD0bEQkkHJbXu+WlTkQxKmnvS13MkbW9ol5TtLp0IZGVEPNH0PolFkq63/ZZOPIy90vZjza40rkFJgxHx8ZF5tU5E0ypNRbJO0vm25488WVsi6cmGdjkl29aJx8ybIuL+pvfJRMQ9ETEnIubpxPf1uYhY2vBaY4qIHZK22V4wctNVklr3gkiRC4tmIuKo7dskPS2pQ9LDEbGxiV0qWCTpJkmv2n5l5LZfRMRTDe70WXK7pJUjPyy3SCpzpddPgI+lAAnecQcSRAIkiARIEAmQIBIgQSRAgkiAxH8Aulp25MdcSe8AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from matplotlib import pyplot as plt\n", - "%matplotlib inline\n", - "plt.imshow(x_normed[np.random.choice(range(1000),10)])" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 2. , 0.98214266, 0.62855945, ..., -0.04959654,\n", - " 1.05254828, -1.32783522],\n", - " [ 2. , -0.60701891, 0.32704136, ..., -0.09251223,\n", - " 1.04318455, -1.32284391],\n", - " [ 1.7826994 , 1.85618152, 1.15562047, ..., -0.02584253,\n", - " 1.03850269, -1.33282653],\n", - " ...,\n", - " [-1.14259331, -0.92485123, -0.09031802, ..., -0.0717345 ,\n", - " 1.77823747, -0.8237132 ],\n", - " [-1.05458292, -0.84539315, -0.04021111, ..., -0.09122515,\n", - " 1.77823747, -0.87362627],\n", - " [-0.78012947, -1.00430931, -0.07044252, ..., -0.04368215,\n", - " 1.75014627, -0.83369581]])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x_normed" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def create_event(event_name):\n", - " class BatchEvent(Event):pass\n", - " BatchEvent.__name__ = event_name\n", - " return BatchEvent\n", - "\n", - "def events(*enames):\n", - " return list(map(create_event,enames))" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class LoopStack(Loop):\n", - " settings = []\n", - " \"\"\"\n", - " A stack of loop\n", - " \"\"\"\n", - " @classmethod\n", - " def from_loops(cls,*loops):\n", - " def init(self,iterable=[],name = None):\n", - " name = name if name!=None else cls.__name__\n", - " \n", - " self.loops = dict()\n", - " l = Loop(iterable)\n", - " for L in loops: l = L(iterable = l)\n", - " super().__init__(iterable = l,\n", - " name = name)\n", - " \n", - " for stuff in cls.settings:\n", - " self.make_stuff(stuff)\n", - " \n", - " setattr(cls,\"init\",init)\n", - " return cls\n", - " \n", - " @classmethod\n", - " def new_setting(cls,*settings):\n", - " cls.settings+=list(settings)\n", - " \n", - " def make_stuff(self,name):\n", - " new_stuff = Stuff(name)\n", - " setattr(self.core,name,new_stuff)\n", - " setattr(self,name,new_stuff)\n", - " \n", - " def __repr__(self,):\n", - " return f\"LoopStack>:{self.name}\\n\\t\"+\\\n", - " \"\\n\\t\".join(map(str,self.core.layers[:-1]))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "TRAIN_EVENTS = [\"DATA_PROCESS\",\"FORWARD\",\"LOSS_CALC\",\n", - " \"BACKWARD\",\"OPT_STEP\",\"METRICS\"]\n", - "EVAL_EVENTS = [\"DATA_PROCESS\",\"EVAL_WRAP\",\"FORWARD\",\"LOSS_CALC\",\"METRICS\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tabulated Metrics" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class MetricTab:\n", - " \"\"\"\n", - " An object handling metric running\n", - " \"\"\"\n", - " def __init__(self):\n", - " self.df_by_epoch = []\n", - " self.mean_by_epoch = []\n", - " self.refresh_list()\n", - " \n", - " def __add__(self,row):\n", - " self.results.append(row)\n", - " return self\n", - " \n", - " def summary(self):\n", - " \"\"\"The summary on metrics\"\"\"\n", - " return pd.DataFrame(self.mean_by_epoch)\n", - " \n", - " def update(self,**batch):\n", - " self.batch.append(batch)\n", - " return self\n", - " \n", - " def result_df(self):\n", - " return pd.DataFrame(self.results)\n", - " \n", - " def batch_df(self):\n", - " return pd.DataFrame(self.batch)\n", - " \n", - " def refresh_list(self):\n", - " self.batch = []\n", - " self.results = []\n", - " \n", - " def calc_mean(self,result_df,batch_df):\n", - " product_df = pd.DataFrame(dict((col,result_df[col]*batch_df[\"bs\"]) for col in self.result_cols))\n", - " batch_mean = dict()\n", - " for col in self.result_cols:\n", - " batch_mean[col] = product_df[col].sum()/batch_df[\"bs\"].sum()\n", - " batch_mean[\"epoch\"] = len(self.mean_by_epoch)+1\n", - " batch_mean[\"span\"] = batch_df.ts.max()-batch_df.ts.min()\n", - " return batch_mean\n", - " \n", - " def mark(self):\n", - " result_df = self.result_df()\n", - " batch_df = self.batch_df()\n", - " \n", - " self.result_cols = result_df.columns\n", - " for col in self.result_cols:\n", - " batch_df[col] = result_df[col]\n", - " \n", - " batch_mean = self.calc_mean(result_df,batch_df)\n", - " \n", - " self.df_by_epoch.append(batch_df)\n", - " self.mean_by_epoch.append(batch_mean)\n", - " \n", - " self.refresh_list()\n", - " \n", - " return batch_df, batch_mean" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "metab = MetricTab()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(100):\n", - " sleep(.01)\n", - " metab+=dict(loss= np.random.rand()*(100-i))\n", - " metab.update(i=i,epoch=i//5,bs = 32,ts = datetime.now())" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
iepochbstsloss
000322020-06-29 23:44:44.96674016.961023
110322020-06-29 23:44:44.97932013.221317
220322020-06-29 23:44:44.99155820.029550
330322020-06-29 23:44:45.00412135.804462
440322020-06-29 23:44:45.01450279.937055
..................
959519322020-06-29 23:44:46.0932001.311746
969619322020-06-29 23:44:46.1032833.787564
979719322020-06-29 23:44:46.1133602.087981
989819322020-06-29 23:44:46.1251220.246656
999919322020-06-29 23:44:46.1378250.154635
\n", - "

100 rows × 5 columns

\n", - "
" - ], - "text/plain": [ - " i epoch bs ts loss\n", - "0 0 0 32 2020-06-29 23:44:44.966740 16.961023\n", - "1 1 0 32 2020-06-29 23:44:44.979320 13.221317\n", - "2 2 0 32 2020-06-29 23:44:44.991558 20.029550\n", - "3 3 0 32 2020-06-29 23:44:45.004121 35.804462\n", - "4 4 0 32 2020-06-29 23:44:45.014502 79.937055\n", - ".. .. ... .. ... ...\n", - "95 95 19 32 2020-06-29 23:44:46.093200 1.311746\n", - "96 96 19 32 2020-06-29 23:44:46.103283 3.787564\n", - "97 97 19 32 2020-06-29 23:44:46.113360 2.087981\n", - "98 98 19 32 2020-06-29 23:44:46.125122 0.246656\n", - "99 99 19 32 2020-06-29 23:44:46.137825 0.154635\n", - "\n", - "[100 rows x 5 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "{'loss': 24.615831910057956,\n", - " 'epoch': 1,\n", - " 'span': Timedelta('0 days 00:00:01.171085')}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch_df,batch_mean = metab.mark()\n", - "display(batch_df)\n", - "batch_mean" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossepochspan
024.615832100:00:01.171085
\n", - "
" - ], - "text/plain": [ - " loss epoch span\n", - "0 24.615832 1 00:00:01.171085" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "metab.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def train_callbacks(loop:Loop)->\"A cluster of callback function\":\n", - " \"\"\"\n", - " call backs allow optimizing model weights\n", - " \"\"\"\n", - " loop.core.metric_tab = MetricTab()\n", - " \n", - " @loop.every_start_FORWARD\n", - " def switch_model_to_train(loop:Loop): loop.model(\"train\")()\n", - " \n", - " @loop.on_DATA_PROCESS\n", - " def opt_zero_grad(loop:Loop):loop.opt(\"zero_grad\")()\n", - " \n", - " @loop.on_BACKWARD\n", - " def opt_move(loop:Loop): \n", - " loop.loss(\"backward\")()\n", - " loop.opt(\"step\")()\n", - " \n", - "def eval_callbacks(loop:Loop):\n", - " loop.core.metric_tab = MetricTab()\n", - " @loop.on_DATA_PROCESS\n", - " def switch_model_to_eval(loop:Loop): loop.model(\"eval\")()\n", - " \n", - "def to_tensor(x):\n", - " return torch.Tensor(x)\n", - " \n", - "def simple_forward(metric_func:list = [])->\"A cluster of callback function\":\n", - " def simple_forward_cb(self):\n", - " @self.on_DATA_PROCESS\n", - " def set_xy(self):\n", - " self.var.clear()\n", - " self.var.x,self.var.y = self.element\n", - " self.var.apply(to_tensor,scope = [\"x\",\"y\"])()\n", - " \n", - " @self.on_FORWARD\n", - " def forward_pass(self):\n", - " y_ = self.model()(self.var.x)\n", - " self.var.y_ = y_.popitem()[1][:,0]\n", - " \n", - " @self.on_LOSS_CALC\n", - " def calculate_loss(self):\n", - " losses = self.loss_func()(self.var.y_,self.var.y)\n", - " self.loss.update(losses)\n", - " \n", - " @self.on_METRICS\n", - " def calcualte_metrics(self):\n", - " # calculate metrics\n", - " with torch.no_grad():\n", - " metrics = self.metric_func()(self.var.y_,self.var.y)\n", - " self.metric.update(metrics)\n", - " \n", - " @self.on_METRICS\n", - " def to_item(self):\n", - " # loop through metrics\n", - " self.metric.update(self.loss.cases)\n", - " dt = dict((k,v.item() if is_tensor(v) else v) \\\n", - " for k,v in self.metric.cases.items())\n", - " self.metric_tab+=dt\n", - " self.pgbar_data(dt)\n", - " \n", - " @self.on_METRICS\n", - " def save_batch_row(self):\n", - " self.metric_tab.update(\n", - " bs = self.var.x.size(0),\n", - " epoch = self.epoch,\n", - " i = self.i,\n", - " ts = datetime.now()\n", - " )\n", - " \n", - " @self.every_end_METRICS\n", - " def show_metric(self):\n", - " self.metric_tab.mark()\n", - " summary =self.metric_tab.summary()\n", - " try:\n", - " from IPython.display import clear_output\n", - " clear_output()\n", - " display(summary)\n", - " except:\n", - " print(summary)\n", - " \n", - " return simple_forward_cb\n", - "\n", - "def single_device(device):\n", - " def single_device_callback(self):\n", - " @self.on_DATA_PROCESS\n", - " def var_to_device(self):\n", - " self.var.apply(\"to\")(device)\n", - " \n", - " @self.before_1st_FORWARD\n", - " def model_to_device(self):\n", - " self.model.apply(\"to\")(device)\n", - " \n", - " return single_device" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - "import torch\n", - "from torch import is_tensor\n", - "\n", - "class TrainLoop(LoopStack):\n", - " def __init__(self,data_iter,model=[],opt=[],loss_func=[],loss=[],hp=[],cuda=[], \n", - " callbacks = [train_callbacks,],tolerate=True):\n", - " loops = [ProgressBar,]\n", - " if tolerate:\n", - " loops.append(Tolerate)\n", - " loops+=list(events(*TRAIN_EVENTS))\n", - " self.from_loops(*loops)\n", - " self.new_setting(\"model\",\"var\",\n", - " \"opt\",\"loss_func\",\"loss\",\n", - " \"hp\",\"cuda\",\"metric_func\",\"metric\")\n", - " self.init(data_iter,)\n", - " for cb in callbacks:\n", - " print(f\"assigning callback {cb}\")\n", - " cb(self)\n", - " \n", - "# @classmethod\n", - "# def from_config(cls,train_data,val_data,model,opt,)\n", - " \n", - "class EvalLoop(LoopStack):\n", - " def __init__(self,data_iter,callbacks,tolerate=False):\n", - " loops = [ProgressBar,]\n", - " if tolerate:\n", - " loops.append(Tolerate)\n", - " loops+=list(events(*EVAL_EVENTS))\n", - " self.from_loops(*loops)\n", - " self.new_setting(\"model\",\"var\",\n", - " \"loss_func\",\"loss\",\n", - " \"hp\",\"cuda\",\"metric_func\",\"metric\")\n", - " self.init(data_iter,)\n", - " \n", - " for cb in callbacks:\n", - " print(f\"assigning callback {cb}\")\n", - " cb(self)\n", - " \n", - " @self.FORWARD.downstream\n", - " def torch_eval_wrap(self,func):\n", - " with torch.no_grad():\n", - " func() " - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [], - "source": [ - "x_train,x_val,y_train, y_val = train_test_split(x_normed,y_normed,test_size = .1)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(18576, 8), (2064, 8), (18576,), (2064,)]" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(a.shape for a in [x_train,x_val,y_train, y_val])" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "assigning callback \n", - "assigning callback .simple_forward_cb at 0x1a2e995950>\n", - "assigning callback \n", - "assigning callback .simple_forward_cb at 0x1a2e995950>\n" - ] - } - ], - "source": [ - "from torch import nn\n", - "import torch\n", - "\n", - "data_train = list(zip(chunkify(x_train),chunkify(y_train)))\n", - "data_val = list(zip(chunkify(x_val),chunkify(y_val)))\n", - "\n", - "forward_cb = simple_forward([nn.MSELoss()])\n", - "\n", - "train_loop = TrainLoop(data_train,tolerate=False,callbacks=[train_callbacks,forward_cb,])\n", - "val_loop = EvalLoop(data_val,callbacks=[eval_callbacks,forward_cb,])" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "@train_loop.every_end_METRICS\n", - "def run_eval(self):\n", - " val_loop.run(1)" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [], - "source": [ - "# loop.search()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Assigning stuff" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [], - "source": [ - "train_loop.model.baseline = nn.Linear(8,1)\n", - "train_loop.loss_func.mse = torch.nn.MSELoss()\n", - "train_loop.metric_func.mae = torch.nn.L1Loss() # MAE is L1 loss \n", - "train_loop.opt.adam = torch.optim.Adam(train_loop.model.baseline.parameters())" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def find_stuff(core:StorageCore):\n", - " klist = list(filter(lambda k:hasattr(getattr(core,k),\"_is_stuff\"), vars(core).keys()))\n", - " return dict((k,getattr(core,k)) for k in klist)\n", - "\n", - "def share_stuff(loop_from:Loop,loop_to:Loop):\n", - " stuff_dict = find_stuff(loop_from.core)\n", - " for k,v in stuff_dict.items():\n", - " setattr(loop_to.core,k,v)" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [], - "source": [ - "share_stuff(train_loop,val_loop)" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4e6efad5dada404d8d12754e2ffa6bc7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(Text(value='', description='KeyWord'), Output()), _dom_classes=('widget-interact',))" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "train_loop.search()" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
maemseepochspan
00.5438400.470426100:00:00.192830
10.4694420.370095200:00:00.218338
20.4359340.328378300:00:00.341273
30.4210580.309691400:00:00.226564
40.4139190.301794500:00:00.370209
50.4107330.298556600:00:00.243021
60.4094060.297206700:00:00.590279
70.4088200.296614800:00:00.211286
80.4085520.296338900:00:00.222686
90.4084070.2962031000:00:00.243863
\n", - "
" - ], - "text/plain": [ - " mae mse epoch span\n", - "0 0.543840 0.470426 1 00:00:00.192830\n", - "1 0.469442 0.370095 2 00:00:00.218338\n", - "2 0.435934 0.328378 3 00:00:00.341273\n", - "3 0.421058 0.309691 4 00:00:00.226564\n", - "4 0.413919 0.301794 5 00:00:00.370209\n", - "5 0.410733 0.298556 6 00:00:00.243021\n", - "6 0.409406 0.297206 7 00:00:00.590279\n", - "7 0.408820 0.296614 8 00:00:00.211286\n", - "8 0.408552 0.296338 9 00:00:00.222686\n", - "9 0.408407 0.296203 10 00:00:00.243863" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "for i in range(10): train_loop.run()" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "from matplotlib import pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
maemseepochspan
00.6382080.643468100:00:03.698205
10.5029920.419795200:00:03.794402
20.4521210.352748300:00:02.817318
30.4280870.322776400:00:03.210466
40.4172100.309802500:00:03.277518
50.4127800.304515600:00:02.883348
60.4109440.302396700:00:03.805901
70.4101530.301516800:00:03.043291
80.4097860.301119900:00:02.914900
90.4095950.3009211000:00:02.387062
\n", - "
" - ], - "text/plain": [ - " mae mse epoch span\n", - "0 0.638208 0.643468 1 00:00:03.698205\n", - "1 0.502992 0.419795 2 00:00:03.794402\n", - "2 0.452121 0.352748 3 00:00:02.817318\n", - "3 0.428087 0.322776 4 00:00:03.210466\n", - "4 0.417210 0.309802 5 00:00:03.277518\n", - "5 0.412780 0.304515 6 00:00:02.883348\n", - "6 0.410944 0.302396 7 00:00:03.805901\n", - "7 0.410153 0.301516 8 00:00:03.043291\n", - "8 0.409786 0.301119 9 00:00:02.914900\n", - "9 0.409595 0.300921 10 00:00:02.387062" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
maemseepochspan
00.5438400.470426100:00:00.192830
10.4694420.370095200:00:00.218338
20.4359340.328378300:00:00.341273
30.4210580.309691400:00:00.226564
40.4139190.301794500:00:00.370209
50.4107330.298556600:00:00.243021
60.4094060.297206700:00:00.590279
70.4088200.296614800:00:00.211286
80.4085520.296338900:00:00.222686
90.4084070.2962031000:00:00.243863
\n", - "
" - ], - "text/plain": [ - " mae mse epoch span\n", - "0 0.543840 0.470426 1 00:00:00.192830\n", - "1 0.469442 0.370095 2 00:00:00.218338\n", - "2 0.435934 0.328378 3 00:00:00.341273\n", - "3 0.421058 0.309691 4 00:00:00.226564\n", - "4 0.413919 0.301794 5 00:00:00.370209\n", - "5 0.410733 0.298556 6 00:00:00.243021\n", - "6 0.409406 0.297206 7 00:00:00.590279\n", - "7 0.408820 0.296614 8 00:00:00.211286\n", - "8 0.408552 0.296338 9 00:00:00.222686\n", - "9 0.408407 0.296203 10 00:00:00.243863" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[,\n", - " ]" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAEvCAYAAAC3wFzvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5fXH8c8JASHIJkSRLQFF3LCiKYpbq6JiVbQurRgrrqkLqFXrT6WtVkXbWq0LuIB1j1q0WnG37tJqJShqwYKIBBAQFGUxICR5fn+cjLMkgQSS3Enm+3695jUz95nl5GYg5z5z7nkshICIiIiIiMRlRR2AiIiIiEi6UZIsIiIiIpJCSbKIiIiISAolySIiIiIiKZQki4iIiIikUJIsIiIiIpIiO+oAUnXr1i3k5+dHHYaIiIiItHDTpk37MoSQW9NY2iXJ+fn5lJSURB2GiIiIiLRwZlZa25jKLUREREREUihJFhERERFJUack2cyGmdksM5tjZpfV8pifmdlMM5thZg8nbK8ws+lVl8kNFbiIiIiISGPZaE2ymbUCxgOHAAuBqWY2OYQwM+Ex/YHLgX1DCF+b2dYJL7EmhLB7A8ctIiIiItJo6jKTPBiYE0KYG0JYBzwKHJ3ymLOA8SGErwFCCEsbNkwRERERkaZTlyS5J7Ag4f7Cqm2JdgB2MLN/mdk7ZjYsYaytmZVUbT9mM+MVEREREWl0dWkBZzVsCzW8Tn/gx0Av4C0z2zWE8A3QJ4SwyMz6Aa+a2UchhE+T3sCsCCgC6NOnTz1/BBERERGRhlWXmeSFQO+E+72ARTU85qkQwvoQwmfALDxpJoSwqOp6LvA6MCj1DUIIE0IIBSGEgtzcGvs5N6riYsjPh6wsvy4ubvIQRERERCSN1CVJngr0N7O+ZtYGOBFI7VLxD+BAADPrhpdfzDWzLma2RcL2fYGZpJHiYigqgtJSCMGvi4qUKIuIiIhkso0mySGEcmAU8CLwMTAphDDDzK42s+FVD3sR+MrMZgKvAb8OIXwF7ASUmNkHVdv/kNgVIx2MGQNlZcnbysp8u4iIiIhkJgshtbw4WgUFBaEpl6XOyvIZ5FRmUFnZZGGIiIiISBMzs2khhIKaxjJ+xb3azhPU+YMiIiIimSvjk+Sf/KR+20VERESk5cv4JPm55+q3XURERERavoxPkufPr992EREREWn5Mj5JVk2yiIiIiKTK+CR57FjIyUnelpPj20VEREQkM2V8klxYCBMmQF6et33Ly/MEeeXKqCMTERERkahkfJIMnijPm+d9kefNgzVr4NxzYcaMqCMTERERkSgoSa7BWWfBFlvA+PFRRyIiIiIiUVCSXINu3WDECHjgAVixIupoRERERKSpKUmuxejR8O23cO+9UUciIiIiIk1NSXIt9tgDfvxjWLo06khEREREpKllRx1AOnvlFcjSYYSIiIhIxlEKuAGxBHnu3GjjEBEREZGmpSR5I/76V9huO/jkk6gjEREREZGmoiR5I444Alq3Vjs4ERERkUyiJHkjuneHE07wLherV0cdjYiIiIg0BSXJdTBqlC9T/eCDUUciIiIiIk1BSXId7L037Lkn3H9/1JGIiIiISFNQC7g6MPNZ5J49o45ERERERJqCkuQ62mmnqCMQERERkaaicot6mDrVV+KbNy/qSERERESkMSlJrofu3eHDD+H226OOREREREQak5LkeujdG445Bu6+G8rKoo5GRERERBqLkuR6Gj0avv4aHnkk6khEREREpLEoSa6nAw6AgQPhttsghKijEREREZHGUKck2cyGmdksM5tjZpfV8pifmdlMM5thZg8nbB9pZp9UXUY2VOBRMYNrr4WLL1aSLCIiItJSbbQFnJm1AsYDhwALgalmNjmEMDPhMf2By4F9Qwhfm9nWVdu3Aq4ECoAATKt67tcN/6M0neHDo45ARERERBpTXWaSBwNzQghzQwjrgEeBo1MecxYwPpb8hhCWVm0/DPhnCGF51dg/gWENE3q0vvkGbrgBPv886khEREREpKHVJUnuCSxIuL+waluiHYAdzOxfZvaOmQ2rx3ObpeXL4f/+D+68M+pIRERERKSh1SVJthq2pVbjZgP9gR8DI4C7zaxzHZ+LmRWZWYmZlSxbtqwOIUWvXz848kiYMAG++y7qaERERESkIdUlSV4I9E643wtYVMNjngohrA8hfAbMwpPmujyXEMKEEEJBCKEgNze3PvFHatQoWLoUHnss6khEREREpCHVJUmeCvQ3s75m1gY4EZic8ph/AAcCmFk3vPxiLvAicKiZdTGzLsChVdtahKFDYcAAbwcnIiIiIi3HRpPkEEI5MApPbj8GJoUQZpjZ1WYW6/PwIvCVmc0EXgN+HUL4KoSwHLgGT7SnAldXbWsRsrJ8cZFOnWD16qijEREREZGGYiHNmv0WFBSEkpKSqMOosxC8d7KIiIiINC9mNi2EUFDTmFbc20yxBHnhQu94ISIiIiLNn5LkBrBkCfTtC+PHRx2JiIiIiDQEJckNoHt3OOgg75m8fn3U0YiIiIjI5lKS3EBGj4ZFi+DJJ6OOREREREQ2l5LkBnL44b7AiNrBiYiIiDR/SpIbSKtWcN558PbbUFoadTQiIiIisjmUJDegM8+EefMgLy/qSERERERkc2RHHUBL0rGjX0D9k0VERESaM80kN7C1a+GQQ+DPf446EhERERHZVEqSG1jbtlBe7j2TKyqijkZERERENoWS5EYwerSfvPf001FHIiIiIiKbQklyIxg+HHr3hnHjoo5ERERERDaFkuRGkJ0N55wDr7wCM2dGHY2IiIiI1Je6WzSSs86C1q2hR4+oIxERERGR+lKS3Ei6dYNLLok6ChERERHZFCq3aEQhwAMPwKRJUUciIiIiIvWhmeRGZAZ33gnLlsHxx0OWDklEREREmgWlbY1s9GiYMwdefDHqSERERESkrpQkN7LjjoPu3eG226KORERERETqSklyI2vTBs4+G55/3meURURERCT9KUluAr/8JRQUeG2yiIiIiKQ/nbjXBLp3h6lTo45CREREROpKM8lNaOVK+OijqKMQERERkY3RTHITGj4cli6FGTO8PZyIiIiIpCfNJDeh006Djz+GV1+NOhIRERER2RAlyU3o5z/35arVDk5EREQkvSlJbkJt28JZZ8HTT8O8eVFHIyIiIiK1qVOSbGbDzGyWmc0xs8tqGD/VzJaZ2fSqy5kJYxUJ2yc3ZPDN0TnneD3ys89GHYmIiIiI1GajJ+6ZWStgPHAIsBCYamaTQwgzUx76txDCqBpeYk0IYffND7Vl6N0b5s6FPn2ijkREREREalOXmeTBwJwQwtwQwjrgUeDoxg2rZYslyBUV0cYhIiIiIjWrS5LcE1iQcH9h1bZUx5nZh2b2uJn1Ttje1sxKzOwdMztmc4JtSX73OxgyBEKIOhIRERERSVWXJLmmjr6pqd3TQH4IYTfgZeD+hLE+IYQC4CTgZjPbrtobmBVVJdIlyzJk7eZevXwVvilToo5ERERERFLVJUleCCTODPcCFiU+IITwVQjhu6q7E4E9E8YWVV3PBV4HBqW+QQhhQgihIIRQkJubW68foEEUF0N+PmRl+XVxcaO/ZWEhdO6sdnAiIiIi6aguSfJUoL+Z9TWzNsCJQFKXCjPbNuHucODjqu1dzGyLqtvdgH2B1BP+olVcDEVFUFrqtQ+lpX6/kRPl9u3hjDPgiSdg4cJGfSsRERERqaeNJskhhHJgFPAinvxOCiHMMLOrzWx41cPON7MZZvYBcD5watX2nYCSqu2vAX+ooStGtMaMgbKy5G1lZb69kZ17LlRWwl13NfpbiYiIiEg9WEizM8cKCgpCSUlJ071hVlbNZ8+ZeQbbyO65Bw48EPr2bfS3EhEREZEEZjat6ty5ajbaJ7nF69PHSyxq2t4ETj+9Sd5GREREROpBy1KPHQs5OcnbcnJ8exOZMgUuvrjJ3k5ERERENkJJcmEhTJgAeXleYpGX5/cLC5sshPffh5tugnffbbK3FBEREZENUE1yGli5Enr2hGOOgQcfjDoaERERkcywoZpkzSSngY4d4dRT4W9/gy++iDoaEREREVGSnCZGjYL1673SQ0RERESipSQ5TQwYAKedBt26RR2JiIiIiKgFXBq5556oIxARERER0Exy2lm/Hl56KeooRERERDKbkuQ0c9ddcNhhMH161JGIiIiIZC4lyWmmsNDXMrnttqgjEREREclcSpLTTJcucPLJ8PDD8NVXUUcjIiIikpmUJKehUaNg7Vq4++6oIxERERHJTEqS09DAgfDjH8Orr0YdiYiIiEhmUgu4NPXYY9C1a9RRiIiIiGQmzSSnqW7dwMzLLkRERESkaSlJTmOvvALbbgszZkQdiYiIiEhmUZKcxnbbDdasgfHjo45EREREJLMoSU5jublw4onwwAOwYkXU0YiIiIhkDiXJaW70aPj2W7j33qgjEREREckcSpLT3J57wpAhMG4cVFZGHY2IiIhIZlALuGbgxhshOxuydEgjIiIi0iSUJDcDQ4ZEHYGIiIhIZtHcZDOxYAGceSZ88knUkYiIiIi0fEqSm4nsbLj/frWDExEREWkKSpKbiW23hRNO8C4Xq1c37nsVF0N+vtdA5+f7fREREZFMoiS5GRk9Glau9L7JjaW4GIqKoLQUQvDroiIlyiIiIpJZ6pQkm9kwM5tlZnPM7LIaxk81s2VmNr3qcmbC2Egz+6TqMrIhg880e+/tLeHGjfMEtjGMGQNlZcnbysp8u4iIiEim2Gh3CzNrBYwHDgEWAlPNbHIIYWbKQ/8WQhiV8tytgCuBAiAA06qe+3WDRJ9hzODSS2HKFF+uOien4d9j/vz6bRcRERFpieoykzwYmBNCmBtCWAc8Chxdx9c/DPhnCGF5VWL8T2DYpoUqAD/7Gdx6a+MkyAB9+tRvu4iIiEhLVJckuSewIOH+wqptqY4zsw/N7HEz612f55pZkZmVmFnJsmXL6hh65goBXn8dFi5s+NceO7Z6Ap6T49tFREREMkVdkmSrYVtqRezTQH4IYTfgZeD+ejyXEMKEEEJBCKEgNze3DiFltsWLYehQuO22hn/twkKYMAHy8ry8Iy/P7xcWNvx7iYiIiKSruiTJC4HeCfd7AYsSHxBC+CqE8F3V3YnAnnV9rtRfjx5wzDFw991em9zQCgth3jyorPRrJcgiIiKSaeqSJE8F+ptZXzNrA5wITE58gJltm3B3OPBx1e0XgUPNrIuZdQEOrdomm2n0aFi+HB5+OOpIRERERFqejSbJIYRyYBSe3H4MTAohzDCzq81seNXDzjezGWb2AXA+cGrVc5cD1+CJ9lTg6qptspkOOAAGDvSSi8ZqByciIiKSqSykWYZVUFAQSkpKog6jWZgwAX77W5g2DXr1ijoaERERkebFzKaFEApqGtOKe83YyJHev1gJsoiIiEjDUpLcjG2xhV8qKqqvkiciIiIim05JcjNXVgY77ADXXx91JCIiIiIth5LkZi4nB3bZxeuTv/tu448XERERkY1TktwCjBoFS5fCpElRRyIiIiLSMihJbgGGDoUBAxpnBT4RERGRTKQkuQXIyvLZ5KlT/SIiIiIimyc76gCkYZxyCuTnwx57RB2JiIiISPOnJLmF6NgRjjwy6ihEREREWgaVW7QgIcCYMapNFhEREdlcSpJbEDMoKYE//AHWr486GhEREZHmS0lyCzN6NCxaBE8+GXUkIiIiIs2XkmSA4mI/6y0ry6+Li6OOaJMdfjj066eSCxEREZHNoSS5uBiKiqC01It6S0v9fjNNlFu1gnPPhSlTYPr0TXyRFnTQICIiIrIplCSPGQNlZcnbysp8ezN1+ulw0kmwxRab8OQWdtAgIiIisikshBB1DEkKCgpCSUlJ071hVpYng6nMoLKy6eJIF/n5nhinysuDefOaOhoRERGRRmNm00IIBTWNaSa5T5/6bW9GZs6Ef/6znk+aP79+20VERERaICXJY8dCTk7ytqws+O1vo4mnAY0aBWedBRUV9XhSCz5oEBEREakrJcmFhTBhgpcTmEFurpdfPPQQrFkTdXSbZdQor5x4+ul6PKmmg4acHN8uIiIikiGUJIMnyvPmeQ3y0qV+ktpWW/mMcjM2fDj07l3PdnCpBw15eX6/sLDR4hQRERFJNzpxrzYheJL4zTew5ZaQnR11RJvk+uvhiivgv/+FXXaJOhoRERGR9KET9zaFmbeCO+AAL+xtpp0uzjzTJ8U/+CDqSERERESaj+Y5PdpUcnLg2GPh97+HDh3glls8eW5GcnPh88+hbduoIxERERFpPpQkb8yVV8KqVXDTTdCxI1x7bdQR1Vvbtl498uWXnjSLiIiIyIYpSd4YM/jzn2HlSu/w0LMnnHNO1FHV2y9/Ca+9BrNmNfvzEUVEREQandKlujCDO++Eyy/3lhHN0IEHwpw58OKLUUciIiIikv7qlCSb2TAzm2Vmc8zssg087ngzC2ZWUHU/38zWmNn0qsudDRV4k2vVCq67zmeSKypgypSoI6qX446D7t3r2Q5OKC72lbqzsvy6uDjqiERERKQpbDRJNrNWwHjgcGBnYISZ7VzD4zoA5wP/SRn6NISwe9Xl7AaIOXp/+hP86Efw979HHUmdtWkDZ58Nzz8Pn3wSdTTNQ3ExFBX5giwh+HVRkRJlERGRTFCXmeTBwJwQwtwQwjrgUeDoGh53DfAnYG0DxpeeRo+GvfeGESPghReijqbOioq83fPEiVFH0jyMGeNdABOVlfl2ERERadnqkiT3BBYk3F9Yte17ZjYI6B1CeKaG5/c1s/fN7A0z23/TQ00jW24Jzz4Lu+4KP/0pvPlm1BHVybbbwksvwVVXRR1J8zB/fv22i4iISMtRlyS5psbA3y/TZ2ZZwF+Ai2t43GKgTwhhEHAR8LCZdaz2BmZFZlZiZiXLli2rW+RR69zZz4LLz4ef/xzWrIk6ojo58EBv/ywb16dP/baLiIhIy1GXJHkh0Dvhfi9gUcL9DsCuwOtmNg/YG5hsZgUhhO9CCF8BhBCmAZ8CO6S+QQhhQgihIIRQkNucGvnm5sLLL8OTT0K7dlFHU2ePPeZNOtJsRfK0M3Zs9QOKnBzfLiIiIi1bXZLkqUB/M+trZm2AE4HJscEQwooQQrcQQn4IIR94BxgeQigxs9yqE/8ws35Af2Bug/8UUerZ0+uTAe65p1mcFVdWBk8/Da++GnUk6a2wECZMgLw87wKYl+f3CwujjkxEREQa20aT5BBCOTAKeBH4GJgUQphhZleb2caaBh8AfGhmHwCPA2eHEJZvbtBp6euv4bLLYOjQtC9a/fnPoVs3GDcu6kjSX2EhzJsHlZV+rQRZREQkM1hIs+/cCwoKQklJSdRhbJr33/ei36239pP5unePOqJaXXEF/PGPMHeuz5CKiIiIZBozmxZCKKhpTCvuNaRBg+C55+Dzz+HQQ2F5+k6ax1bWvv32aOMQERERSUdKkhvaPvvAU0/B7Nm+ckea6t0brrwSDjgg6khERERE0o+S5MYwdKifwJfmBay/+x0ccUTUUaQ5rUstIiKSkZQkN5beVV3zpkzxlfnWrYs2nlosWQLjx6sdXI20LrWIiEjGUpLc2GbNgkcfhZNPhvLyqKOp5umnYdQo+Ne/oo4kDWldahERkYylJLmxnXEG3Hijr+Bx1lneSyyNnHSSLx54221RR5KGtC61iIhIxlKS3BQuugiuugruuw8uvDCtahvat/c8/u9/96YckkDrUouIiGQsJclN5Xe/82R56VKoqIg6miTnnusT3HfeGXUkaUbrUouIiGQsJclNxQz+/Gc/6Ss7G9asiTqi7/XrB8OH+0l8kkDrUlejZh8iIpIptOJeFJYu9QbFo0fDeedFHQ3gk9utWkUdhaSzWLOPxHMZc3Iy/rhBRESaMa24l266dIEBA7ytxP33Rx0NEE+QdU6a1EbNPkREJJMoSY5C69bwt7/BwQfD6af7WXNp4KmnvKLg3XejjkTSkZp9iIhIJlGSHJW2bT0r3XtvX2zklVeijoiDDoIOHdQOTmqmZh8iIpJJlCRHqX17ePZZT5IHDow6Gjp0gFNPhUmT4Isvoo5G0s3YsXBq62I+I58KsviMfE5tXaxmHyIi0iIpSY5a585el7z11rB+va/QF6HzzvMVtAcMUAcDSVZIMROtiHxKySKQTykTrYhC9AEREZGWR0lyOrnwQhgyBD76KLIQSko8OV6xwtc8KS31jgZKlIUxY8hel3zmXva6zD1zT+3wRERaNrWASyeffQb77ef92N56C/r3b/IQ8vM9MU6Vlwfz5jV1NJJWsrJqXi3SLO2WW29saocnItIyqAVcc9G3L7z8sifJQ4dG0jZAHQykVjpz73tqhyci0vIpSU43O+0EL73k9Q7HHVfzzF0jqi3fad8eVq9u0lAk3WiZ7u/Nnw8jSD6JcQTFGX0wqfITEWlplCSno0GD4Lnn4I47/KvsJlRTHpSd7QnyoEHwn/80aTiSTrRM9/dGbVXMRFJOYqSIUVtlZmYYKz8pLdW5DCLScihJTlf77AMFVSUyDz0Eq1Y1ydvWlAfddx+8+aY33xg50qtBJEMVFnpxemWlX2dgggxwHWNoT3K9RXvKuI7MrLcYMwaOLkueWT+6rDijy080sx6nfSHNlZLkdDd7tjcvPuqo6kWQjaSmPGj//eGDD+DJJ30J6zVr/DxDkUy05fKa6ypq297S7Vta88z6vqWZmQ0VF8PLpxXzemk+5SGL10vzefm04oxMDrUvqtNBQ7J03h9KktPdDjvAgw/6VO7xx3sT44h06uQl0wBXXQW77eazzGnWIEWk8ekkxiR/bFXzzPofW2XmVPJ/Lihm3Prkg4Zx64v4zwVp9Ne/iWhfJNNBQ7J03x9qAddcTJzoRX7HHw+PPOKFwhGaPx9OOQXeeMNDuvNO6No10pBEmo56wCUJloVR/W9JwLCQWe0BAeZZPvlU76U5jzzyw7ymDidS80z7ItH53Yq5/quipIPKb8nh8q4TuPXLzPu/Ix32x4ZawClJbk7+8he46CJvE3fwwVFHQ0UF3Hgj/OY3kJsLjz3mpdQiGaG42Itx58/3GeSxYzMyQQbUYD1FpWWRVcNBQyVGVoYdNGhfJJtnOmhINM+i3x/qk9xS/OpXXhicBgkyeG3ypZd6x4sePWCrraKOSKQJ6STGOLUHTFLWteaym9q2t2TaF8n6UPN5C7Vtb+nSfX8oSW5udtvNr19+Ga6+OtpYqgwaBO++Czvu6PXJv/lNpCtri0hTU3vAJFveMpbyNskHDeVtctjylsw7aNC+SKaDhmTpvj/qlCSb2TAzm2Vmc8zssg087ngzC2ZWkLDt8qrnzTKzwxoiaAEmT4Yrr4Q//CHqSIB4O+fFi+Huu7173V/+knGrFYtkLs2sxxUWkn1P8kFD9j0ZetCgfZFEBw3J0n1/bLQm2cxaAbOBQ4CFwFRgRAhhZsrjOgDPAm2AUSGEEjPbGXgEGAz0AF4Gdggh1NppVzXJdVRR4WfOPfwwjBsH550XdUTfW7oUzjwTnn4aDjnEO2D06BF1VCIiImlA5zMki3h/bNaJe2Y2BLgqhHBY1f3LAUII16c87mY8Cb4EuKQqSU56rJm9WPVab9f2fkqS62H9ejjhBHjqKc9ER46MOqLvheANOX71K+jVC2bO9BpmERERkXSxuSfu9QQWJNxfWLUt8Q0GAb1DCM/U97myGVq3hkcfhaFDYcqUqKNJYuYdst5/H26/3RPkykpf3lpEREQk3dUlSbYatn0//WxmWcBfgIvr+9yE1ygysxIzK1m2bFkdQpLvtW3r9cl33eX302zN6B12iDfjuPFG+MEP4O1av0cQERERSQ91SZIXAr0T7vcCFiXc7wDsCrxuZvOAvYHJVSfvbey5AIQQJoQQCkIIBbm5ufX7CQTatfP1HOfN8yz0jTeijqhGQ4Z4Dr///r5iX3l51BGJiIiI1KwuSfJUoL+Z9TWzNsCJwOTYYAhhRQihWwghP4SQD7wDDA8hlFQ97kQz28LM+gL9gXcb/KcQ1769Z6FHHuk92dLMfvt5m+eTToLf/97vz5kTdVQiIiIi1W00SQ4hlAOjgBeBj4FJIYQZZna1mQ3fyHNnAJOAmcALwHkb6mwhmyk31/sn5+bCsGFp2ay4Uyd44AEvpZ4zB5YsiToiERERkeq0LHVLNHeu1zRUVPgJfdtvH3VENVq9Grbc0m8/+CAcfjh06xZtTCIiIpI5tCx1punXz2eU99wzrdeKjiXIn38OZ53liwm+9FK0MYmIiIiAkuSWa6ed4NlnPUn+7jtI464hPXvCO+9Aly5w2GFwwQWwZk3UUYmIiEgmU5KcCUaM8D5sy5dHHUmtdt8dSkpg9Gi49VY44IC062YnIiIiGURJciY47zyYNctP5lu1KupoatWunSfIzz/vIcdW6EuzsnkRERHJAEqSM8HBB8Njj8F778FRR0FZWdQRbdCwYXDqqX77kUd8QcGFCyMNSURERDKMkuRMMXy4t5B4800/S66ZKC/3euXddvM8X0RERKQpKEnOJCNGwH33wf/9X9SR1NkvfgHTp0P//vCzn8HIkbByZdRRiYiISEunJDnTnHKKT8uGAJMnQ2Vl1BFtVP/+3u75t7+Fhx6CV1+NOiIRERFp6ZQkZ6pXXoGjj/Z+a83gzLjWreHqq+F//4NjjvFt//43rF8fbVwiIiLSMilJzlQHHwwXXwzjxsGYMVFHU2f9+/v1woVw0EGw337wySfRxiQiIiItj5LkTGUGN9wARUVw/fV+aUZ69fLSi08+8R7LEyc2iwlxERERaSaUJGcyM7j9djjpJPjNb7yWoRk5/nj46CMYMsRz/eOOaxYl1iIiItIMKEnOdK1aeceL116DHXeMOpp669kTXnoJbroJdt0VsvSJFhERkQaglEL8rLgDDvDbzz7rJ/Pl53vGmZ8PxcVRRrdRWVnwq1/5iX0Ar78O558Pa9ZEGpaIiIg0Y0qSJS4E+PWvfW3o0lK/X1rqtQxpnignevttuO022HNP77EsIiIiUl9KkiXODFatqr69rKxZdcC4/HIvwfjmG00ljiIAAB6lSURBVBg82M9PVK2yiIiI1IeSZEn2+ec1b58/v2nj2EyHHOIn9R11FFx6KUyaFHVEIiIi0pwoSZZkffrUvv3Xv4bHH4fy8qaNaRN17erhPvOML2kNtR8DiIiIiCRSkizJxo6FnJzkbTk5vib0k0/CCSfA9tvDjTd6PUOaM4MjjvCT+xYtgoED4Re/gBUroo5MRERE0pmSZElWWAgTJkBenmeYeXl+/4wzYNYs+Mc/oG9fuOQSX9Hj5ZejjrjOtt7aG3c88gj84Afw1ltRRyQiIiLpykKaLVNWUFAQSkpKog5DNmb6dF/S+oYboEsX77tWWQkHHujJdRp75x0/Fpg3Dy67DK65Rv2VRUREMpGZTQshFNQ0ptRANs3uu8Pdd3uCDL6s9cEH+/Z774W1a6ONbwP23ttz/FNP9Q53SpBFREQkldIDaRhPPQV//av3Vj79dC/TmDgx6qhq1aGDh3vffX5/xgy46y4PX0RERERJsjSMtm09Of7gA3jlFW9QnJ3tY6tW+fY0FAtxwgQ4+2w4+mhYujTamERERCR6SpKlYZnBQQfB00/Daaf5tnvu8TKMgw6CyZOhoiLaGGvwl7/AzTf7IiQDB8Jzz0UdkYiIiERJSbI0vlNO8RP8Pv3Up2oHDPB1o9OotiEryztfTJ0K22zjbeOeeMJX487P9/H8/Ga1OreIiIhshjolyWY2zMxmmdkcM7ushvGzzewjM5tuZlPMbOeq7flmtqZq+3Qzu7OhfwBpBrp08ZZxn37qS99tsw38/e/xLhjLl0cbX4KBA+Hdd+GPf/QqkaIiP7kvBL8uKlKiLCIikgk22gLOzFoBs4FDgIXAVGBECGFmwmM6hhBWVt0eDpwbQhhmZvnAMyGEXesakFrAZYhVq/zsuSVLoF8/OPxwuPBC2G+/tGkhl5/viXGqvDxvHyciIiLN2+a2gBsMzAkhzA0hrAMeBY5OfEAsQa7SHkif79ElPXXo4NfZ2Z4cv/46HHAA/PCH8NBDsG5dpOEBzJ9f8/bSUiXJIiIiLV1dkuSewIKE+wurtiUxs/PM7FPgT8D5CUN9zex9M3vDzPbfrGil5enWDa67DhYsgDvvhG+/hZEj4fPPfTzCuuU+fWof++ILv54xwxcdXL++aWISERGRplGXJLmm776rZS4hhPEhhO2A/wN+U7V5MdAnhDAIuAh42Mw6VnsDsyIzKzGzkmXLltU9emk5cnLgl7/0rHPqVF/6GuBnP/NC4BkzmjyksWM9rNQw77rLJ7wB7rgDDjnEy6xPOQWefBLKypo8VBEREWlgdUmSFwK9E+73AhZt4PGPAscAhBC+CyF8VXV7GvApsEPqE0IIE0IIBSGEgtzc3LrGLi1RVhbssYffrqyErl3hwQdh113h0EO9N1tlZZOEUljo/ZPz8rxMOi/P7xcVxVfp+9OfPDE+6ih45hk49ljYZZf4BHgaVI2IiIjIJqjLiXvZ+Il7BwOf4yfunRRCmJHwmP4hhE+qbh8FXBlCKDCzXGB5CKHCzPoBbwEDQwi1tjPQiXtSzZdfenY6bhwsXgy33ALnn7/x5zWx9evhrbe8FGPECE+Ut9vOTwD86U/hmGOgd++NvoyIiIg0kc06cS+EUA6MAl4EPgYmhRBmmNnVVZ0sAEaZ2Qwzm46XVYys2n4A8KGZfQA8Dpy9oQRZpEbdusEVV/jZcg89BCed5NsnT4bLLvN65jTQurWvlzJihN//7ju/vWSJ5/R9+niZxjPPRBuniIiIbNxGZ5KbmmaSpc5+9zsvHDaDE07wLhl77RV1VDWaNQv+8Q8vzbjiChg+HD7+GO67z2eZBw+Ol3CIiIhI09jQTLKSZGne5s3z1fvuvhtWroRf/AIeeCDqqOrkwQfh9NOhvBx69PDFCH/6UzjwQO+MJyIiIo1rc/ski6Sv/Hy48UZYuBBuvRWOPNK3f/utb//660jD25Bf/AKWLvVkecgQuP9+PwFw7Vofnz0bVq+ONkYREZFMpZlkaZmeeAKOO857tp16KlxwAexQrbFKWlmzBj74APbe2+/vtZffP/RQn2E+6igvzxYREZGGoZlkyTzHHgvvv+99lu++GwYM8Fnmb7+NOrJatWsXT5DB28v98peeKJ9+uvdi/r//iy4+ERGRTKIkWVqu3XeHe+/19aWvvNILfdu397G33orXNaSpH/3Iu93NmwfTpvkJf4MH+9iSJVBQANdcA//9b6QLE4qIiLRIKreQzPPVV9CzJ3TsCGefDeeeC927Rx1VvXz4oYf+9tt+f/vtvQ/zhRf6jyYiIiIbp3ILkURbbQXPP++1Ddde6w2MR46EuXOjjqzOdtsN/v1vWLQI7rzTFy259db4YoT//je88IJW/BMREdlUmkmWzPbJJ95C7r77vKahf3+fae7cGVq1ijq6elm9Grbc0m+fcAI8/rhPlv/kJ37i3+GHQ4cO0cYoIiKSTjSTLFKb/v19CnbJEr8NcNZZfvvmm/2kv/x8X+kjPx+Ki6OMdoNiCTJ4W7mnn/Zk+ZVX4Oc/h4MPjo+n8fmLIiIiaUFLFoiAt4qLOflkb2D8q18lP6a0FIqK/HZhYdPFtgnatvVmHkceCRUVXn5RVuZja9b44iU/+IHPMB9zDPTtG228IiIi6UYzySKpjj0Wpkyp+WS+sjI45xyfqp050zPQNNeqFey/Pxx2mN//7jvP/1esgIsugn79vBHIq69GG6eIiEg6UZIsUpsvvqh5+6pVcMopsMsuXvS7zz7wzjs+tnYtrF/fdDFugs6d4aqrvP/ynDm+MGGHDv6jgHfHu/hiP05oBscAIiIijUJJskht+vSpfft//+vrSJ91lvdfjp0R9+ijfnuvvXzGeeJEPyGwvLzp4q6H7bbz2eS33vK+ywDTp8O4cT773KOH/4jPPecJc3FxsynRFhER2SzqbiFSm+Jir0GOFfOC1y5PmFB7TXJJCfztb/Dee3755hvf/sUXsPXWnm3Omwd77OFFwe3aNfqPsSlWrvQueU8+6SG3bw833OArANZnd4iIiKSzDXW30Il7IrWJZX5jxviqfX36wNixG84ICwriU7IhwGefwUcfeYIMnkA/8IDfbtUKdtrJyzXuusu3VVSkReu5jh29I8bPf+41zHPmwBFHJCfI4PfPPNOPBc47z7fNng29eiWfCykiItLcaCZZpCmFAAsW+CzztGl+DfDss379ox95Z4099/TZ5j32gEGDoFOn6GKukpVV+/LXl1ziM80VFT45vn49bLutl3Nstx0cdxwcdZQ//6uvoGtXMGva+EVERFJpJlkkXZj5jHSfPt57LdWRR/oZc2+8ES/4PeIIeOYZv33HHb4G9R57eKbZhPr08S54qfLyPEEGX/HvgQfg00/jl5dfhh128CR58eL4iuCxBLpfP+/nXFAQP1EwDSbTRUQkw2kmWSRdffEFvP++1y0ccIB31ejUKT6dm5fnyfIZZ3gi3cg2pUQ7JgQ/Pli+vHoS/dlnfn7jyJHw7ruw335+UmAsid5uO+/Kl5fXqD+eiIhkIM0kizRH22wDw4bF73foAF9+6YlzYrnGwoU+Pns2HHSQJ86J5Ro9ejRIbcOmlGjHxN5+q63gwguTxyoq4jPIXbt6t41YAv3vf/tJhD/4gSfJTzwBo0cnJ9DbbedLbnfuvNk/ooiIyPc0kyzS3MWmaWfNgmuu8cT5f/+LzzhPnuy1DnPnen+3PfbwjLMZFAXHapi33NJXEXz7bZ+5jiXRixb542bP9pXEJ06EO++Ml3HEkuj99oM2baL9WUREJP1oJlmkJYsluwMGwEMP+e3Vq321kPfeg8GDfdvkyfGltrfaKj7jfOmlfj8NmUG3bvH7Q4b4JaaszMs1Ystqd+7sjUSmT4d//CO+rkusROSGG+C116rPRO+8c9P8PCIi0nxoJlkkU6xZ4+3oYj2cp02Djz/2Eo6cHK+dePnl5HKN/v2Tz6IrLt60eosIVFR4JUppqZd0gyfJjzzis9ArV/q2bt1g2TK//fvfexvrxAR6++1rPoZoRrtCRERqoZlkEfHebIMHx2eWwadaW7f22506+ZTr+PHeHBmSW1pceSX86U++9Db49qIiv52G2WGrVl5VknjC369/7ZdYGcenn8LXX8fHFy6EF1/0LhwxgwbFO/X95jeefC9b5pP2sd2U5rtCREQ2gWaSRSTZ+vVe0zxtmnfUGD3at7dpE69fSJSX59Ov99/vCXdurtc85Ob6NG0zLAYuK/MS7k8/9f7QRx3l2/fZB6ZOrX2V8S228GOQjh390qkT/PjHvigLeGK95Zbx8Y4dfVfppEMRkWhsaCZZSbKI1E1tq4mYeYPkrbZKnpYFn1aN1UkPHeodOnJz44n0XnvB3nv76y5a5NvTPKkuL/cQa/uv88c/9lKOlSthxQo4+WS46SZYt86T6FSXXgp//KM/drvtkhPojh3htNN8MZYVK3ySP3V8p5184ZaKCj+G2WKLZnFOpohIWlC5hYhsvtpWE+nTx68/+cTrEJYu9etly+Jn1JWXe1Y5Z463qPjyS8/qLr3Uk+SVK30ta/Dp11gife65nmWuWgV//Wtygh27jpWLNJHs7A0vrPLaa7U/b86ceAIdS6J33DH+mBNP9G2x8cWL47XTS5Z4DXSqO+6As8/2cvNBg3x3xGaxO3aE66/3ToKzZ8PNNycn2J06wcEHe5fAVau8NXdsrL7Jtmq0RaSlUZIsInUzdmzNq4mMHeu3u3b1S2LWF5OdDa+8Er9fWQnffBPPwrKzvXdbYoK9dKnPXoMv5R3rzJHo9tvhnHP8BMTTT48nz7EE+qij/My7sjJfySQ3t+bp3E3YFS+fVsyV68fQh/nMpw+/bz2WoWNrzwqzsnymuDadOsG4cbWPDxjg5eCJSfbKlfHX3HprT4hjyXdsfMstfXzxYnj8cd8Wq6UGr8Hu0QP++U+fsY5p3dpjevZZLyH55z/h1lvjyXfsctZZ8NJLfr1mjT+3tBTOPNMT+wsu8F/vmjXxEvg2bVr+qoo6aBBp/upUbmFmw4BbgFbA3SGEP6SMnw2cB1QAq4GiEMLMqrHLgTOqxs4PIby4ofdSuYVIGovqL38sqY4l0LHL/vt7vcFHH3kSnThWXh7vEf3ss77kN3hmF0ukx4/3Lh4zZsALL1RPsrfd1jO8GvZD+elFZK+LHzCUt8kh+546LD+YBr77Lp5Ed+8O7dv7r/TNN5NnuVeu9Mn+vDxvqXf11cnj69b5DPUhh9Q8sw7w+eeehF91lXcPicnK8oR52TKvwrnmGrj7bk+gY5fWreE///FjqVtv9UQ9lmS3aePPGz/eX+/hh/3XGBtv3Rq6dPFkHXyG/4svkl+/Uyev+AGf5V+3Lvn127WL14vH2pHXxeasTtkS6YAhmfZHsqj3x2bVJJtZK2A2cAiwEJgKjIglwVWP6RhCWFl1ezhwbghhmJntDDwCDAZ6AC8DO4QQKmp7PyXJIrLZQvCkul07X4WktNSnTBOT6KVL4ZZbvEnyxInx9hSJPvwQBg70/8Vvuy2ePMemZFNtuy28/nr8fbt1a9EFwt9950lldnbtNdplZb473n7bL+vW+WX9er++9lp/jUcf9eOUxLHycj++Af/D+cQTyc9v2xZmVv0lKiyESZOST6rs1cu/hABflfGFF5Jj23FH/xICfMGZf/0refyHP/Sl0gF23x3++994At6mDRx4oL9n7PWXLPHtH3yQPFsf066dryCfleUz6fvv71+EgJfMVFTEx7Ky4Ec/ghNO8O2XX+7bEsf3399L/deu9Y9y6viQIf4zrF7t+zd1vKDAv6FYudK/6Ekcy8ryj37Pnj4+fXr18e239wOR1av9n1jqePfu8OSTyd8yxPbDTTfBGWf4vly3zsfNki/t2vnrxFblTBwDH2tu/7x0AJUsHfbH5ibJQ4CrQgiHVd2/HCCEcH0tjx8BnBJCODz1sWb2YtVrvV3b+ylJFpEmF4JPjSYm0MuWeZFwhw7w97/DXXfFx2JL/W3M2rVe3nHRRT5N2q5d/NKhQzwDu/lmmDIlPta2rZeuXHmljz//vGd7ieNdusC++/r4ggWeRbRtG39M69ZNlkHk58M+pcVcR7z85ArG8u+8QubNa5IQvldZ6YlyLMmOzQR//rkne7EEPDZr/MMf+vgbb/hMc2IS3rUrHH+8j99+u//aE5P4/v3jy6yffbaPr19fPRlPtPPO/quqrISjj/be3eBlM2vXxhPCykqvILrhBn+vzp19W2ysshIuu8xLbJYv91hTXXutz9CVlvrvKNUtt8D553vyP3Bg9fF77vETR99+2zu7pHrsMd8/L70Ehx1Wffz5532/1PYtw5Qp/hG+/3449dTq4x98ALvt5mVIsSY7iebO9dMe/vAHuOKK6kn2okV+nHrVVb4fU8e/+ML/yVx2mR8nJ461aeMtIcH/+U6alDzepYsfOACMGuW/89hYVpYfXMQqzH75Sz8Aiy2MWlOToDZt/HcQe41dd/X9D54sfvqp346NDx7s/22AH0gtXhz/527mveGvvdbvH320//eWOH7oof5zA/zkJx5T4vhRR/nPVV4e7+6TOH7CCf47W7UKTjqp+vjJJ/tjli2Lzz8kjp9+uh8w9url/zZTxZomNYXNPXGvJ7Ag4f5CYK8a3uQ84CKgDXBQwnPfSXluzzq8p4hI0zHzLKRzZ898Uh13XHLBbl6efzeYqls3/8u1Zo1fYp069tvPE/HY9rVrk6dely71Kc3Y+Jo1HkssSb79dnjmmeT32n57P1kSYOTI6mcM7r47vP++3x4+3F8/MUnfc0+48UYf/+1vvTNJYpK9445w7LE+/vTTnpXFEvR27WCbbb4/afORgyfyg3suIAefLsynlIkU8f5PAJp2eiwrK14ukahnT7/U5kc/2vDrnnvuhsfvvDN+e0MHDTNm1Pz8WBJUkzZtkmfaYmIfoS5dfDw1iW7b1sd79vTjqMTxigr/YgQ8QZ8+Pfm5FRXxevedd/aEL3V8jz18fLfdPIlMfO3KSk/65s+HEVTfF49QSL9+/vwf/tA/iiEkX7bd1sf33tsTvtj22M8eOwDaZx8/GEh9frt2Pr7XXj5jnzgG8UqqPff02fDE8djpEOAJ64oVyePt28fH+/f3fz6J44kHLb16+Yx9CH5AUuP+WFfINtvEY0tsC9mxo99PjD32uwU/Do+dahEbT4w/tj3x+RUJ3+eXlfmBWOLzY+3wQ/CDsMT9Dr6/wH/PixZVH4990bZ+vR/MpI5/841fL1pU8/54dH56TKvXZSb5BOCwEMKZVfd/AQwOIdRwXAdmdlLV40ea2Xjg7RDCQ1VjfwWeCyH8PeU5RUARQJ8+ffYsre2wU0QkHTT1d4QrVvhfpcQk28y/LwdfKXHBAt8ee0y3bj6FBfC733nRbeLzBw70EhLwJP5//4uPh+DTT//4h49vs40n8olOOsn3A9TeHjAvz7O/nj09I4ldWrXy7+AvucSnog46KHksO9unMAsLvRNKUVH18ZNO8mLoJUt8KjG2PXZ9zDGexS1e7AXLse2xxxx8MPTr589/883k57Zq5ZlTbq6vOjN7dvJ4drZnwjk58O23niElvPbbv36C3f56Pu2Jfz6+JYf3z5nAfrenxx//pnJ+t2Ku/6qo2r64vOsEbv0ys/YFaH+kSof9sbkzyQuB3gn3ewEb+q7xUeCO+jw3hDABmABeblGHmEREohNLhJvqbJNOnfxSm6FDN/z8q6/e8PiUKfHbIfi0UuJU05tvejKYmGR37578nJrMnx9PuCsq/Lvb2CU2TQiejCaOr10b/0563TqfMS8vjz+mosILcsET1PvuSx4vL/ckdo89/DvbSy6pHtukSZ4kf/hhfLWXRM8/773z3ngj+VuEmLfe8oOLxx+vViswpIZd0Z4y9rvjZLjrFD/AmT3b3//Pf/aDmNj39LFC2zlz/EDnuuviBcexx5j5Pmnb1s+GfPDB5PFYYTT4txFPPZU83rmznwUJ/i3C668nj3fv7gcW4MtMvvdecmx5eX4mZez5s2cnx9a/v9c4AH8uO482lFXbF9cxBij0n/2rr5KLjXff3QuWY+//7bfJ4wUFMGKE3x4zxn/fieNDhvi3JxUV38eRNB4r5i4r8/2fOG7mzc733dcPTu+4o/rzhw71z9aXX8IDD1R//tChsMsufgD2+ONJ4zes/S1b1LA/blh7Pkys2j5sGPTu7Z/dxK5Asfc/4gg/cJ0zJ/nfbmx8+HD/euF//4uXdCWO//Sn3vZmxox4zUji+LHH+mfrww/jRf+J48cf7weM77/vMSSOZ2XFv4GaNi251iZWx3LEEX5/6lT+tO5XtN3Q5yNidUmSpwL9zawv8DlwInBS4gPMrH8Ioep7P44AYrcnAw+b2U34iXv9gXcREWnuCgtb5pk2ZtXb5A0YsOHn5OXV3kM7O9vruWvToQM891zt4z16ePeS2uy0U/y720SxxH3wYP/uN5Y8xxLprbby8X328WQhdTzWynDffT1hTkzQE8eHDPFvEBIT9JraFcZccYV/R92li9/fc08v/ozVMYSQXCuxyy6edKSOx3ro9e3r9QiJ44nftXft6r+f2PNSawVatfLi7Nh4rKA7JlarHxuvrEzu+PLpp56QJ75+wlmLbdasqHE3bLm8qlzp2Wf9s5P4ffyKFfEkubjYk+jE8VWr4knyuHH+fonj55wTT5Kvuy65zgC8GHfoUD/gi5U0JRo71n/v33zjZ0ymuvVWT5KXLIGLL64+fs89/nv77LNqxdS1NaDc4tvl8eLdF17wJPm99+LtWRL961+eJP/rX/6NS6oPPvDP18sv117MveWW3v3niiuqjy9b5p+/Rx/1ovdUa9f65+bee+PfRsXEzsQE/93cd1/yeJcuXr8B8Mc/0nbVsuqvT8LnI2J1bQH3E+BmvAXcPSGEsWZ2NVASQphsZrcAQ4H1wNfAqBDCjKrnjgFOB8qBC0MIz2/ovXTinohIM5MOp6ink/z82lebaeozGaOWjvsi1s8vtYg5sSA51rIlloAnJuGxFicVFT7LnVrwm5PjB5rr11cvZi4oiJ8RmKhnT+93CH5g07at/3v66qv4+8Zss42//urVPpudOt6jh4+vWFHz8/v08fiXL695vF8///mXLat5fMAA30eLF8cT3sTxXXf16wUL4quwJhZLx84S/ewzPxhZvLj6/mjCz4eWpRYRkcYVdbPTdKKDhjjti2TaH8nSYH9sKEnOqmmjiIhIvRQW+sxPZaVfZ+If/JjCQv8jn5cXr+HN1CRI+yKZ9keyNN8fmkkWERERkYykmWQRERERkXpQkiwiIiIikkJJsoiIiIhICiXJIiIiIiIplCSLiIiIiKRQkiwiIiIikkJJsoiIiIhICiXJIiIiIiIp0m4xETNbBtSw0HuT6AZ8GdF7S3rTZ0Nqo8+GbIg+H1IbfTbSQ14IIbemgbRLkqNkZiW1rboimU2fDamNPhuyIfp8SG302Uh/KrcQEREREUmhJFlEREREJIWS5GQTog5A0pY+G1IbfTZkQ/T5kNros5HmVJMsIiIiIpJCM8kiIiIiIimUJANmNszMZpnZHDO7LOp4JH2YWW8ze83MPjazGWZ2QdQxSXoxs1Zm9r6ZPRN1LJI+zKyzmT1uZv+r+v9jSNQxSXows19V/T35r5k9YmZto45JapbxSbKZtQLGA4cDOwMjzGznaKOSNFIOXBxC2AnYGzhPnw9JcQHwcdRBSNq5BXghhLAj8AP0GRHAzHoC5wMFIYRdgVbAidFGJbXJ+CQZGAzMCSHMDSGsAx4Fjo44JkkTIYTFIYT3qm6vwv/Q9Yw2KkkXZtYLOAK4O+pYJH2YWUfgAOCvACGEdSGEb6KNStJINtDOzLKBHGBRxPFILZQke8KzIOH+QpQESQ3MLB8YBPwn2kgkjdwMXApURh2IpJV+wDLg3qpSnLvNrH3UQUn0QgifA38G5gOLgRUhhJeijUpqoyQZrIZtavkhScxsS+DvwIUhhJVRxyPRM7MjgaUhhGlRxyJpJxvYA7gjhDAI+BbQ+S6CmXXBv63uC/QA2pvZydFGJbVRkuwzx70T7vdCX31IAjNrjSfIxSGEJ6KOR9LGvsBwM5uHl2kdZGYPRRuSpImFwMIQQuxbp8fxpFlkKPBZCGFZCGE98ASwT8QxSS2UJMNUoL+Z9TWzNngB/eSIY5I0YWaG1xV+HEK4Kep4JH2EEC4PIfQKIeTj/2+8GkLQjJAQQlgCLDCzAVWbDgZmRhiSpI/5wN5mllP19+VgdFJn2sqOOoCohRDKzWwU8CJ+luk9IYQZEYcl6WNf4BfAR2Y2vWrbFSGE5yKMSUTS32iguGryZS5wWsTxSBoIIfzHzB4H3sO7J72PVt5LW1pxT0REREQkhcotRERERERSKEkWEREREUmhJFlEREREJIWSZBERERGRFEqSRURERERSKEkWEREREUmhJFlEREREJIWSZBERERGRFP8PRyKICpVYrNwAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.figure(figsize = (12,5))\n", - "metric_df = train_loop.metric_tab.summary()\n", - "val_metric_df = val_loop.metric_tab.summary()\n", - "\n", - "display(metric_df)\n", - "display(val_metric_df)\n", - "plt.plot(metric_df.mse,\"bo--\",label=\"train mse\")\n", - "plt.plot(metric_df.mae,\"bo\",label=\"train mae\")\n", - "plt.plot(val_metric_df.mse,\"ro--\",val_metric_df.mae,\"ro\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/20_config.ipynb b/nbs/20_config.ipynb deleted file mode 100644 index 2339b65..0000000 --- a/nbs/20_config.ipynb +++ /dev/null @@ -1,362 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Config" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp config" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import json\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class Config(dict):\n", - " def __setattr__(self,k,v):\n", - " self[k]=v\n", - " \n", - " def __getattr__(self,k,):\n", - " return self[k]\n", - " \n", - " def __delattr__(self,k):\n", - " del self[k]\n", - " \n", - " def pretty_print(self):\n", - " print(json.dumps(self,indent = 4))\n", - " \n", - " def __call__(self,**kwargs):\n", - " \"\"\"\n", - " assign more keyword value\n", - " \"\"\"\n", - " self.update(kwargs)\n", - " return self\n", - " \n", - " def save(self,json_path,indent=None):\n", - " \"\"\"\n", - " save to json file\n", - " \"\"\"\n", - " with open(json_path,\"w\") as f:\n", - " json.dump(self,f,indent = indent)\n", - " \n", - " @classmethod\n", - " def load(cls,path):\n", - " \"\"\"\n", - " load from json file\n", - " \"\"\"\n", - " with open(path,\"r\") as f:\n", - " obj = cls()(**json.loads(f.read()))\n", - " return obj\n", - " \n", - " def first(self,key):\n", - " return first(self,key)\n", - " \n", - " def getall(self,key):\n", - " return getall(self,key)\n", - " \n", - "def first(d,key):\n", - " if hasattr(d,\"items\"):\n", - " for k,v in d.items():\n", - " if k==key: return v\n", - " else: \n", - " ans = first(v,key)\n", - " if ans!=None: return ans\n", - " \n", - " if type(d) in [tuple,list,set]:\n", - " for i in d:\n", - " ans = first(i,key)\n", - " if ans!=None: return ans\n", - " return None\n", - "\n", - "def getall(d,key):\n", - " results = []\n", - " if hasattr(d,\"items\"):\n", - " for k,v in d.items():\n", - " if k==key: results+=[v,]\n", - " else: results += getall(v,key)\n", - " \n", - " if type(d) in [tuple,list,set]:\n", - " for i in d:\n", - " results += getall(i,key)\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 106, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dict.items" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"a\": 1,\n", - " \"z\": 3,\n", - " \"b\": 2,\n", - " \"c\": [\n", - " {\n", - " \"d\": 5,\n", - " \"e\": 6,\n", - " \"z\": 5\n", - " },\n", - " {\n", - " \"g\": 7,\n", - " \"h\": {\n", - " \"i\": 8,\n", - " \"z\": 6\n", - " }\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], - "source": [ - "config = Config(a = 1,z=3,b=2,c=[dict(d=5,e=6,z=5),dict(g=7,h=dict(i=8,z=6))])\n", - "config.pretty_print()" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "config.first(\"z\")" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[3, 5, 6]" - ] - }, - "execution_count": 109, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "config.getall(\"z\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pretty print config" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"a\": 1,\n", - " \"b\": 2\n", - "}\n" - ] - } - ], - "source": [ - "Config(a=1,b=2).pretty_print()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multiple ways of setting config" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "config = Config(a=1,b=2)(c=5)(d=6)\n", - "config.e=12" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'a': 1, 'b': 2, 'c': 5, 'd': 6, 'e': 12}" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "config" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save and load json" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"a\": 1, \"b\": 2, \"c\": 5, \"d\": 6, \"e\": 12}" - ] - } - ], - "source": [ - "config.save(\"test.json\")\n", - "!cat test.json" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'a': 1, 'b': 2, 'c': 5, 'd': 6, 'e': 12}" - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Config.load(\"test.json\")" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "!rm test.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/21_minimum.ipynb b/nbs/21_minimum.ipynb deleted file mode 100644 index 86ba617..0000000 --- a/nbs/21_minimum.ipynb +++ /dev/null @@ -1,463 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Minimum\n", - "> Minimal way of managing a **(Directed Acyclic Graph)DAG pipeline**" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp minimum" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from forgebox.config import Config\n", - "from datetime import datetime\n", - "from uuid import uuid4\n", - "from inspect import getargspec\n", - "from copy import deepcopy" - ] - }, - { - "cell_type": "code", - "execution_count": 155, - "metadata": { - "code_folding": [ - 22, - 57, - 64 - ] - }, - "outputs": [], - "source": [ - "# export\n", - "def now():return datetime.now().timestamp()\n", - "\n", - "class Link:\n", - " is_link_identifier = True\n", - " data = Config()\n", - " inputs = Config()\n", - " verbose = 0\n", - " def __init__(self,func):\n", - " self.ensure_funcs()\n", - " func_name = func.__name__\n", - " self.func_name = func_name\n", - " self.funcs_[func_name] = func\n", - " self.func = self.funcs_[func_name]\n", - " self.register_args()\n", - " \n", - " def __repr__(self,):\n", - " return f\"[Link Function]:{self.func}\"\n", - " \n", - " def ensure_funcs(self,):\n", - " if hasattr(self,\"funcs_\")==False:\n", - " self.__class__.funcs_ = Config()\n", - " \n", - " def ensure_uuid(self,uuid:str)->str:\n", - " \"\"\"\n", - " Make sure there's uuid,\n", - " if not create on\n", - " \"\"\"\n", - " if uuid == False:\n", - " uuid = str(uuid4())\n", - " self.data[uuid] = Config()\n", - " self.inputs[uuid] = Config()\n", - " return uuid\n", - " else:\n", - " return uuid\n", - " \n", - " def check_callback(self,v,uuid):\n", - " \n", - " if hasattr(v,\"is_link_identifier\")==False:\n", - " # this is not a callback function\n", - " return v\n", - " \n", - " elif v.func_name in self.data[uuid]:\n", - " # found calculated result of the function\n", - " # read the cached result\n", - " if self.verbose>0:\n", - " print(f\"[{self.func_name}] using cache:\\t{v.func_name}\")\n", - " return self.data[uuid].get(v.func_name)\n", - " \n", - " else:\n", - " # calculated result of function v not found,\n", - " # execuate the calculation\n", - " if self.verbose>0:\n", - " print(f\"[{self.func_name}] execute callback:\\t{v.func_name}\")\n", - " result = v.unit_call(uuid)()\n", - " return result\n", - " \n", - " @staticmethod\n", - " def value_only(dict_:Config)->Config:\n", - " \"\"\"\n", - " filter out the value that requires callback\n", - " \"\"\"\n", - " return Config((k,v) for k,v in dict_.items() if hasattr(v,\"is_link_identifier\")==False)\n", - " \n", - " @staticmethod\n", - " def callback_only(dict_:Config)->Config:\n", - " \"\"\"\n", - " filter out the value that does not require callback\n", - " \"\"\"\n", - " return Config((k,v) for k,v in dict_.items() if hasattr(v,\"is_link_identifier\")==True)\n", - " \n", - " def get_cached(self,uuid:str)->Config:\n", - " \"\"\"\n", - " Get the cached data\n", - " \"\"\"\n", - " this_data = self.data[uuid]\n", - " return Config((k,this_data.get(k)) for k in self.all_argname \\\n", - " if k in this_data)\n", - " \n", - " def unit_call(self,uuid = False):\n", - " uuid = self.ensure_uuid(uuid)\n", - " \n", - " def called(*args,**kwargs):\n", - " # record inputs\n", - " kwargs0 = deepcopy(self.kwargs_default)\n", - " kwargs0.update(kwargs)\n", - " \n", - " # save inputs\n", - " self.inputs[uuid].update(Config({self.func_name:Config(args = args,\n", - " kwargs = kwargs0)}))\n", - " # save data\n", - " self.data[uuid].update(self.value_only(kwargs0))\n", - " \n", - " # check the cache for all args\n", - " cached = self.get_cached(uuid)\n", - " kwargs0.update(cached)\n", - " \n", - " # check the callback trigger\n", - " args0 = list(self.check_callback(a,uuid) for a in args)\n", - " kwargs0 = dict((k,self.check_callback(v,uuid)) for k,v in kwargs0.items())\n", - " \n", - " # redundant keywords\n", - " if (len(args0)>0) and len(self.arg_spec.args)>=len(args0):\n", - " manual_args = self.arg_spec.args[:len(args0)]\n", - " for k in manual_args:\n", - " if k in kwargs0:\n", - " del kwargs0[k]\n", - " if len(args0)>len(manual_args):\n", - " # TODO: continue development\n", - " for arg in args0[-len(manual_args):]:\n", - " pass\n", - " \n", - " # run function\n", - " if self.verbose > 1:\n", - " print(f\"[{self.func_name}] final args {args0}\")\n", - " print(f\"[{self.func_name}] final kwargs {kwargs0}\")\n", - " rt = self.func(*args0,**kwargs0)\n", - " \n", - " # save outputs\n", - " if hasattr(rt,'items'):\n", - " self.data[uuid].update(rt)\n", - " self.data[uuid].update({self.func_name:rt})\n", - " \n", - " return rt\n", - " return called\n", - " \n", - " def __call__(self,*args,**kwargs,):\n", - " return self.unit_call()(*args,**kwargs)\n", - " \n", - " def register_args(self)->None:\n", - " \"\"\"\n", - " Register all the arguements\n", - " \"\"\"\n", - " arg_spec = getfullargspec(self.func)\n", - " self.arg_spec = arg_spec\n", - " \n", - " # gather all the default **keyword** arguments\n", - " kwargs_default = dict()\n", - " if arg_spec.defaults != None:\n", - " kwargs_default.update(dict(zip(arg_spec.args[::-1],arg_spec.defaults[::-1])))\n", - " if arg_spec.kwonlydefaults != None:\n", - " kwargs_default.update(dict(zip(arg_spec.kwonlyargs[::-1],arg_spec.kwonlydefaults[::-1])))\n", - " self.kwargs_default = kwargs_default\n", - " \n", - " # gather all the arg,kwarg name\n", - " self.all_argname = []\n", - " if len(arg_spec.args)>0:\n", - " self.all_argname+=arg_spec.args\n", - " if len(arg_spec.kwonlyargs)>0:\n", - " self.all_argname+=arg_spec.kwonlyargs" - ] - }, - { - "cell_type": "code", - "execution_count": 156, - "metadata": {}, - "outputs": [], - "source": [ - "Link.verbose = 2" - ] - }, - { - "cell_type": "code", - "execution_count": 159, - "metadata": {}, - "outputs": [], - "source": [ - "@Link\n", - "def abc(a = 2,b=3):\n", - " return a**2\n", - "\n", - "@Link\n", - "def process_abc(f = abc,d=2): \n", - " return f*d\n", - "\n", - "@Link\n", - "def step2(d=5,e = process_abc,):\n", - " return {\"power\":e**d,\"mod\":e%d}\n", - "\n", - "@Link\n", - "def step3(mod):\n", - " return mod/2\n" - ] - }, - { - "cell_type": "code", - "execution_count": 160, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[step3] final args []\n", - "[step3] final kwargs {}\n" - ] - }, - { - "ename": "TypeError", - "evalue": "step3() missing 1 required positional argument: 'mod'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstep3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munit_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mcalled\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"[{self.func_name}] final args {args0}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"[{self.func_name}] final kwargs {kwargs0}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m \u001b[0mrt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;31m# save outputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: step3() missing 1 required positional argument: 'mod'" - ] - } - ], - "source": [ - "step3()" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "25" - ] - }, - "execution_count": 91, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "abc(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'9ddc067c-20ae-4489-88cb-facaf968b2c2': {'abc': {'args': (5,),\n", - " 'kwargs': {'b': 3, 'a': 2}}}}" - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "abc.inputs" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'9ddc067c-20ae-4489-88cb-facaf968b2c2': {'b': 3, 'a': 2, 'abc': 25}}" - ] - }, - "execution_count": 93, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "abc.data" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'abc': }" - ] - }, - "execution_count": 94, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "abc.funcs_" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "from inspect import getargspec,getfullargspec" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [], - "source": [ - "def a():return 123\n", - "def b(a,b=1):return 123\n", - "def c(*args,a=1,b=2):return a\n", - "def d(a=1,b:{\"type\":int}=2,**kwargs, ):return a\n", - "def e(a=1,**kwargs):return a" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "FullArgSpec(args=[], varargs=None, varkw=None, defaults=None, kwonlyargs=[], kwonlydefaults=None, annotations={})\n", - "FullArgSpec(args=['a', 'b'], varargs=None, varkw=None, defaults=(1,), kwonlyargs=[], kwonlydefaults=None, annotations={})\n", - "FullArgSpec(args=[], varargs='args', varkw=None, defaults=None, kwonlyargs=['a', 'b'], kwonlydefaults={'a': 1, 'b': 2}, annotations={})\n", - "FullArgSpec(args=['a', 'b'], varargs=None, varkw='kwargs', defaults=(1, 2), kwonlyargs=[], kwonlydefaults=None, annotations={'b': {'type': }})\n", - "FullArgSpec(args=['a'], varargs=None, varkw='kwargs', defaults=(1,), kwonlyargs=[], kwonlydefaults=None, annotations={})\n" - ] - } - ], - "source": [ - "for f in [a,b,c,d,e]:\n", - " print(getfullargspec(f))" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [], - "source": [ - "arg_spec = getfullargspec(b)\n", - "\n", - "kwargs_default = dict()\n", - "if arg_spec.defaults != None:\n", - " kwargs_default.update(dict(zip(arg_spec.args[::-1],arg_spec.defaults[::-1])))\n", - "if arg_spec.kwonlydefaults != None:\n", - " kwargs_default.update(dict(zip(arg_spec.kwonlyargs[::-1],arg_spec.kwonlydefaults[::-1])))" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'b': 1}" - ] - }, - "execution_count": 76, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "kwargs_default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/30_traceable_edit_in_flask.ipynb b/nbs/30_traceable_edit_in_flask.ipynb deleted file mode 100644 index e7f3db7..0000000 --- a/nbs/30_traceable_edit_in_flask.ipynb +++ /dev/null @@ -1,323 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 01 A logged editable table\n", - "> Traceable editable table in flask" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from flask import Flask\n", - "app = Flask(__name__)\n", - "\n", - "@app.route('/')\n", - "def hello_world():\n", - " return 'Hello, World!'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run a simple applitcation" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp editable" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import pandas as pd\n", - "from datetime import datetime\n", - "import json\n", - "from sqlalchemy import create_engine as ce\n", - "from sqlalchemy import text\n", - "from jinja2 import Template" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from pathlib import Path\n", - "def get_static():\n", - " import forgebox\n", - " return Path(forgebox.__path__[0])/\"static\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def edit_js():\n", - " with open(get_static()/\"edit.js\",\"r\") as f:\n", - " return f\"\"\n", - "\n", - "\n", - "class DefaultTemp(Template):\n", - " \"\"\"\n", - " Jinjia template with some default render config\n", - " \"\"\"\n", - " def render(self,dt):\n", - " dt.update(dict(type=type,now = datetime.now()))\n", - " return super().render(dt)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create sample data" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "con = ce(\"sqlite:///sample.db\")\n", - "\n", - "sample_df = pd.DataFrame(dict(name=[\"Darrow\",\"Virginia\",\"Sevro\",]*20,\n", - " house =[\"Andromedus\",\"Augustus\",\"Barca\"]*20,\n", - " age=[20,18,17]*20))\n", - "\n", - "sample_df.to_sql(\"sample_table\",index_label=\"id\",\n", - " index=True,\n", - " con = con, method='multi',\n", - " if_exists=\"replace\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from flask import request\n", - "from flask import g\n", - "from datetime import datetime\n", - "\n", - "class Editable:\n", - " def __init__(self,name,app,table_name,con,id_col,\n", - " log_con,log_table=\"editable_log\",columns = None):\n", - " \"\"\"\n", - " name: route name for url path, \n", - " also it will be the task title appearning on the frontend\n", - " app:flask app\n", - " table_name: table to edit\n", - " con:sqlachemy connnection, created by : con = sqlalchemy.create_engine\n", - " id_col: a column with unique value\n", - " log_con:sqlachemy connnection, for storaging change log\n", - " \"\"\"\n", - " self.name = name\n", - " self.app = app\n", - " self.table_name = table_name\n", - " self.con = con\n", - " self.log_con = log_con\n", - " self.columns = \",\".join(columns) if columns!=None else \"*\"\n", - " self.id_col = id_col\n", - " \n", - " self.t_workspace = self.load_temp(get_static()/\"workspace.html\")\n", - " self.t_table = self.load_temp(get_static()/\"table.html\")\n", - " self.assign()\n", - " \n", - " def assign(self): \n", - " self.app.route(f\"/{self.name}\")(self.workspace)\n", - " self.app.route(f\"/{self.name}/df_api\")(self.read_df)\n", - " self.app.route(f\"/{self.name}/save_api\",\n", - " methods=[\"POST\"])(self.save_data)\n", - "\n", - " def workspace(self):\n", - " return self.t_workspace.render(dict(title=self.name,\n", - " pk=self.id_col,\n", - " edit_js = edit_js()))\n", - "\n", - " def save_data(self):\n", - " data = json.loads(request.data)\n", - " # update change and save log\n", - " changes = data[\"changes\"]\n", - " log_df = pd.DataFrame(list(self.single_row(change) for change in changes))\n", - " \n", - " log_df[\"idx\"] = log_df.idx.apply(str)\n", - " log_df[\"original\"] = log_df.original.apply(str)\n", - " log_df[\"changed\"] = log_df.changed.apply(str)\n", - " log_df.to_sql(f\"editable_log\",con = self.log_con,index=False, if_exists=\"append\")\n", - " \n", - " print(log_df)\n", - " # return updated table\n", - " query = data[\"query\"]\n", - " page = query[\"page\"]\n", - " where = query[\"where\"]\n", - " return self.data_table(page,where)\n", - " \n", - " def settype(self,k):\n", - " if k[:3] == \"int\": return int\n", - " elif \"float\" in k: return float\n", - " elif k==\"str\":return str\n", - " elif k==\"list\":return list\n", - " elif k==\"dict\":return dict\n", - " else: return eval(k)\n", - " \n", - " def single_row(self,row):\n", - " row[\"ip\"]= request.remote_addr\n", - " row[\"table_name\"] = self.table_name\n", - " row[\"ts\"] = datetime.now() \n", - " if row[\"original\"]==row[\"changed\"]: \n", - " row['sql'] = \"\"\n", - " return row\n", - " else:\n", - " col = row[\"col\"]\n", - " val = row[\"changed\"] \n", - " val = f\"'{val}'\" if 'str' in row[\"valtype\"] else val\n", - " idx = row[\"idx\"]\n", - " idx = f\"'{idx}'\" if type(idx)==str else idx\n", - " set_clause = f\"SET {col}={val}\"\n", - " sql = f\"\"\"UPDATE {self.table_name} \n", - " {set_clause} WHERE {self.id_col}={idx}\n", - " \"\"\"\n", - " row['sql'] = sql\n", - " self.con.execute(sql)\n", - " return row\n", - " \n", - " def read_df(self):\n", - " page = request.args.get('page')\n", - " where = request.args.get('where')\n", - " return self.data_table(page,where)\n", - " \n", - " def data_table(self,page,where):\n", - " where_clause = \"\" if where.strip() == \"\" else f\"WHERE {where} \"\n", - " sql = f\"\"\"SELECT {self.columns} FROM {self.table_name} {where_clause}\n", - " ORDER BY {self.id_col} ASC LIMIT {page},20\n", - " \"\"\"\n", - " print(sql)\n", - " df = pd.read_sql(sql,self.con)\n", - " df = df.set_index(self.id_col)\n", - " return self.t_table.render(dict(df = df))\n", - " \n", - " def load_temp(self,path):\n", - " with open(path, \"r\") as f:\n", - " return DefaultTemp(f.read())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Testing editable frontend" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "app = Flask(__name__)\n", - "\n", - "# Create Editable pages around sample_table\n", - "Editable(\"table1\", # route/task name\n", - " app, # flask app to wrap around\n", - " table_name=\"sample_table\", # target table name\n", - " id_col=\"id\", # unique column\n", - " con = con,\n", - " log_con=con\n", - " )\n", - "\n", - "app.run(host=\"0.0.0.0\",port = 4242,debug=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieve the log" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.df import PandasDisplay" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with PandasDisplay(max_colwidth = 0,max_rows=100):\n", - " display(pd.read_sql('editable_log',con = con))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/40_wild_tree_and_loss.ipynb b/nbs/40_wild_tree_and_loss.ipynb deleted file mode 100644 index 9bbb50e..0000000 --- a/nbs/40_wild_tree_and_loss.ipynb +++ /dev/null @@ -1,885 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Wild Tree and loss\n", - "> A tree loss for classifiying categories as a tree structure" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp wildtree" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import pandas as pd\n", - "import numpy as np\n", - "from pathlib import Path\n", - "from collections import Counter\n", - "from typing import Callable, List, Dict" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sample data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We down load the data from [oncotree here](http://oncotree.mskcc.org/api/tumorTypes/tree?&version=oncotree_latest_stable), save as json file" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "with open(Path(os.environ['HOME'])/\"Downloads\"/\"oncotree.json\",\"r\") as f:\n", - " oncotree = json.load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['code', 'color', 'name', 'mainType', 'externalReferences', 'tissue', 'children', 'parent', 'history', 'level', 'revocations', 'precursors'])" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "oncotree['TISSUE'].keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Wildtree" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def cache(f: Callable) -> Callable:\n", - " \"\"\"\n", - " cache for class property, use as decorator\n", - " \"\"\"\n", - " fname = f.__name__\n", - "\n", - " def wrapper(self):\n", - " if fname in self.caches:\n", - " return self.caches[fname]\n", - " else:\n", - " v = f(self)\n", - " self.caches[fname] = v\n", - " return v\n", - " return wrapper\n", - " \n", - "class WildNode:\n", - " \"\"\"\n", - " A node in graph, with uncertain edge possibilities\n", - " \"\"\"\n", - " def __init__(self, name):\n", - " self.name = name\n", - " self.nodes[name] = self\n", - " self.as_parent = []\n", - " self.as_kid = []\n", - " self.all_edges = []\n", - " self.caches = dict()\n", - "\n", - " @classmethod\n", - " def get(cls, name):\n", - " if name in cls.nodes:\n", - " return cls.nodes[name]\n", - " else:\n", - " return cls(name)\n", - "\n", - " def __repr__(self) -> str:\n", - " return self.name\n", - "\n", - " @property\n", - " @cache\n", - " def kids(self):\n", - " return list(e.kid for e in self.as_parent)\n", - "\n", - " @property\n", - " @cache\n", - " def parents(self):\n", - " return list(e.parent for e in self.as_kid)\n", - "\n", - " @property\n", - " @cache\n", - " def kid_summary(self):\n", - " return dict(Counter(map(lambda x: x.name, self.kids)))\n", - "\n", - " @property\n", - " @cache\n", - " def parent_summary(self):\n", - " return dict(Counter(map(lambda x: x.name, self.parents)))\n", - "\n", - " @property\n", - " @cache\n", - " def detail(self) -> str:\n", - " return f\"\\n\\tparents:{self.parent_summary}\\n\\tkids:{self.kid_summary}\"\n", - " \n", - " @classmethod\n", - " def calc_to_root(cls, root):\n", - " for node in cls.nodes.values():\n", - " path = node.search(root)\n", - " node.to_root = len(path)-1\n", - "\n", - " def search(self, another: str) -> List[Dict[str, str]]:\n", - " \"\"\"\n", - " Search a treval parth within a possible tree\n", - " - another: str, name\n", - " \"\"\"\n", - " fresh_step = {\"name\": self.name, \"direction\": \"none\", \"freq\":\"1\"}\n", - " searched = dict({self.name: {\"path\": [fresh_step]}})\n", - " unsearched = self.nodes.keys()\n", - " height = []\n", - " latest_depth = [self.name, ]\n", - "\n", - " def conclude(\n", - " latest_depth: List[Dict[str, str]],\n", - " new_depth: List[Dict[str, str]]\n", - " ):\n", - " latest_depth.clear()\n", - " latest_depth += new_depth\n", - "\n", - " def deeper():\n", - " new_depth = []\n", - " for name in latest_depth:\n", - "\n", - " obj = self.get(name)\n", - " for direction, summary in\\\n", - " zip(\n", - " [\"up\", \"down\"],\n", - " [obj.parent_summary, obj.kid_summary]):\n", - " for n, freq in summary.items():\n", - " if n in searched:\n", - " continue\n", - "\n", - " new_step = {\"name\": n, \"freq\": freq,\n", - " \"direction\": direction}\n", - " searched[n] = dict(\n", - " path=searched[name][\"path\"]+[new_step, ])\n", - " new_depth.append(n)\n", - " conclude(latest_depth, new_depth)\n", - " if another == n:\n", - " return\n", - " conclude(latest_depth, new_depth)\n", - "\n", - " while True:\n", - " if len(latest_depth) == 0:\n", - " return []\n", - " if another in latest_depth:\n", - " return searched[another]['path']\n", - "\n", - " deeper()\n", - "\n", - "class WildEdge:\n", - " def __init__(\n", - " self,\n", - " parent: WildNode,\n", - " kid: WildNode,\n", - " ):\n", - " self.parent = parent\n", - " self.kid = kid\n", - " for node in [parent, kid]:\n", - " node.all_edges.append(self)\n", - " parent.as_parent.append(self)\n", - " kid.as_kid.append(self)\n", - " self.edges.append(self)\n", - "\n", - " def __repr__(self):\n", - " return f\"[parent:{self.parent}][kid:{self.kid}]\"\n", - " \n", - "class WildTree:\n", - " \"\"\"\n", - " A tree that will analyze a tree structure\n", - " from (parent-kid) pairs data\n", - " use tree.create_new_edge('parent','kid') to \n", - " add a new edge\n", - " \n", - " The tree doesn't have to be a very clear structure,\n", - " 2 nodes can have both parent-kid and siblings relationship\n", - " the tree will find the shortest path anyway\n", - " \n", - " ### **Important**\n", - " After added all the edges**, use ```tree.root_map``` for\n", - " finding the tree root.\n", - " \n", - " tree('name1','name2') to get a dataframe on travel path\n", - " \"\"\"\n", - " def __init__(self):\n", - " self.node_class,self.edge_class = self.new_class()\n", - " self.create_new_edge=self.edge_class.from_names\n", - " self.nodes = self.node_class.nodes\n", - " \n", - " def find_a_root(self):\n", - " for k, node in self.nodes.items():\n", - " if len(node.parents)==0 and len(node.kids)!=0:\n", - " yield node\n", - " \n", - " def __repr__(self):\n", - " return f\"tree.node_class,tree.create_new_edge('a', 'b')\"\n", - " \n", - " def __getitem__(self, node_name:str):\n", - " return self.nodes[node_name]\n", - " \n", - " def new_class(self):\n", - " class TreeNode(WildNode):\n", - " nodes = dict()\n", - " class TreeEdge(WildEdge):\n", - " nodes = TreeNode.nodes\n", - " edges = list()\n", - " @classmethod\n", - " def from_names(cls, parent: str, kid: str,):\n", - " return cls(\n", - " TreeNode.get(parent),\n", - " TreeNode.get(kid))\n", - " \n", - " TreeEdge.nodes = TreeNode.nodes\n", - " return TreeNode,TreeEdge\n", - " \n", - " def root_map(self):\n", - " \"\"\"\n", - " Necessary step!\n", - " Run this after input all the edges\n", - " \"\"\"\n", - " self.root = next(self.find_a_root())\n", - " self.node_class.calc_to_root(self.root.name)\n", - " \n", - " def __call__(self, a: str, b: str) -> pd.DataFrame:\n", - " \"\"\"\n", - " Calculate the travel path between 2 nodes\n", - " \"\"\"\n", - " df = pd.DataFrame(self[a].search(b))\n", - " df['to_root'] = df.name.apply(lambda x:self[x].to_root)\n", - " return df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test wildtree" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def parse_node(node: WildNode, callback: Callable) -> None:\n", - " parent = node['name']\n", - " if 'children' in node:\n", - " for child_name,child_node in node['children'].items():\n", - " # this is the onlything you have to do to fill a \n", - " # wild tree data, create new edge for \n", - " # every parent/kid string pairs\n", - " callback(parent, child_node['name'])\n", - " parse_node(child_node, callback=callback)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "tree = WildTree()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "parse_node(oncotree['TISSUE'], callback=tree.create_new_edge)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "tree.root_map()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "WildTree found the root node automatically" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Tissue" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tree.root" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tissue\tOvary/Fallopian Tube\tOvarian Cancer, Other\tOvarian Choriocarcinoma, NOS\tHigh-Grade Neuroendocrine Carcinoma of the Ovary\tHigh-Grade Serous Fallopian Tube Cancer\tOvarian Epithelial Tumor\tEndometrioid Borderlin Ovarian Tumor\tSerous Ovarian Cancer\tHigh-Grade Serous Ovarian Cancer\tLow-Grade Serous Ovarian Cancer\tBrenner Tumor\tBrenner Tumor, Benign\tBrenner Tumor, Borderline\tBrenner Tumor, Malignant\tClear Cell Borderline Ovarian Tumor\tOvarian Seromucinous Carcinoma\tOvarian Seromucinous Adenoma\tEndometrioid Ovarian Cancer\tSerous Borderline Ovarian Tumor\tMucinous Ovarian Cancer\tSerous Borderline Ovarian Tumor, Micropapillary\tMixed Ovarian Carcinoma\tMucinous Borderline Ovarian Tumor\tClear Cell Ovarian Cancer\tSmall Cell Carcinoma of the Ovary\tOvarian Seromucinous Borderline Tumor\tOvarian Carcinosarcoma/Malignant Mixed Mesodermal Tumor\tOvarian Germ Cell Tumor\tImmature Teratoma\tPolyembryoma\tMixed Germ Cell Tumor\tMature Teratoma\tDysgerminoma\tEmbryonal Carcinoma\tYolk Sac Tumor\tSex Cord Stromal Tumor\tSertoli-Leydig Cell Tumor\tSteroid Cell Tumor, NOS\tGranulosa Cell Tumor\tGonadoblastoma\tFibrothecoma\tLymphoid\tLymphoid Neoplasm\tPosttransplant Lymphoproliferative Disorders\tClassical Hodgkin Lymphoma PTLD\tPolymorphic PTLD\tFlorid Follicular Hyperplasia PTLD\tMonomorphic PTLD (B- and T-/NK-cell types)\tPlasmacytic Hyperplasia PTLD\tInfectious Mononucleosis PTLD\tB-Lymphoblastic Leukemia/Lymphoma\tB-Lymphoblastic Leukemia/Lymphoma with Recurrent Genetic Abnormalities\tB-Lymphoblastic Leukemia/Lymphoma, BCR-ABL1 Like\tB-Lymphoblastic Leukemia/Lymphoma with t(v;11q23.3);KMT2A Rearranged\tB-Lymphoblastic Leukemia/Lymphoma with t(1;19)(q23;p13.3);TCF3-PBX1\tB-Lymphoblastic Leukemia/Lymphoma with t(5;14)(q31.1;q32.3) IL3-IGH\tB-Lymphoblastic Leukemia/Lymphoma with iAMP21\tB-Lymphoblastic Leukemia/Lymphoma with Hyperdiploidy\tB-Lymphoblastic Leukemia/Lymphoma with Hypodiploidy\tB-Lymphoblastic Leukemia/Lymphoma with t(9;22)(q34.1;q11.2);BCR-ABL1\tB-Lymphoblastic Leukemia/Lymphoma with t(12;21)(p13.2;q22.1); ETV6-RUNX1\tB-Lymphoblastic Leukemia/Lymphoma, NOS\tNon-Hodgkin Lymphoma\tMature B-Cell Neoplasms\tSolitary Plasmacytoma of Bone\tFollicular Lymphoma\tIn Situ Follicular Neoplasia\tDuodenal-Type Follicular Lymphoma\tMonoclonal B-Cell Lymphocytosis\tLymphoplasmacytic Lymphoma\tWaldenstrom Macroglobulinemia\tBurkitt Lymphoma\tPrimary Cutaneous DLBCL, Leg Type\tEBV Positive Mucocutaneous Ulcer\tDLBCL Associated with Chronic Inflammation\tB-Cell Prolymphocytic Leukemia\tPrimary DLBCL of the central nervous system\tB-Cell Lymphoma, Unclassifiable, with Features Intermediate between DLBCL and Classical Hodgkin lymphoma\tAlpha Heavy-Chain Disease\tChronic Lymphocytic Leukemia/Small Lymphocytic Lymphoma\tMu Heavy-Chain Disease\tPrimary Cutaneous Follicle Center Lymphoma\tLymphomatoid Granulomatosis\tHigh-Grade B-Cell Lymphoma, NOS\tMonoclonal Gammopathy of Undetermined Significance\tIgG\tIgA\tIgM\tBurkitt-Like Lymphoma with 11q Aberration\tHHV8 Positive DLBCL, NOS\tPrimary Mediastinal (Thymic) Large B-Cell Lymphoma\tPlasmablastic Lymphoma\tGamma Heavy-Chain Disease\tEBV Positive DLBCL, NOS\tLarge B-Cell Lymphoma with IRF4 Rearrangement\tExtraosseous Plasmacytoma\tPediatric-Type Follicular Lymphoma\tHairy Cell Leukemia\tHigh-Grade B-Cell Lymphoma, with MYC and BCL2 and/or BCL6 Rearrangements\n" - ] - } - ], - "source": [ - "print('\\t'.join(list(tree.nodes.keys())[:100]))" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namedirectionfreqto_root
0Chromophobe Renal Cell Carcinomanone14
1Renal Non-Clear Cell Carcinomaup13
2Renal Cell Carcinomaup12
3Kidneyup11
4Tissueup10
5Head and Neckdown11
6Salivary Carcinomadown12
7Pleomorphic Adenomadown13
\n", - "
" - ], - "text/plain": [ - " name direction freq to_root\n", - "0 Chromophobe Renal Cell Carcinoma none 1 4\n", - "1 Renal Non-Clear Cell Carcinoma up 1 3\n", - "2 Renal Cell Carcinoma up 1 2\n", - "3 Kidney up 1 1\n", - "4 Tissue up 1 0\n", - "5 Head and Neck down 1 1\n", - "6 Salivary Carcinoma down 1 2\n", - "7 Pleomorphic Adenoma down 1 3" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tree('Chromophobe Renal Cell Carcinoma','Pleomorphic Adenoma')" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\tparents:{'Renal Non-Clear Cell Carcinoma': 1}\n", - "\tkids:{}\n" - ] - } - ], - "source": [ - "print(tree['Chromophobe Renal Cell Carcinoma'].detail)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Integrate to binary cross entropy\n", - "This part will create loss function with weights (according to tree), nhot encoder to translate string of a branch" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def calc_weight(x: np.array):\n", - " \"\"\"\n", - " Calculate the weight for BCELoss,\n", - " where the nodes closer to root will cause bigger loss\n", - " when it's wronged confidently\n", - " \"\"\"\n", - " exp = np.exp(-x)\n", - " return exp/exp.sum()\n", - "\n", - "def loss_package(tree: WildTree) -> dict:\n", - " \"\"\"\n", - " Create an entire package of things,\n", - " Input:\n", - " - tree: WildTree\n", - " Output:\n", - " - dictionary keys:\n", - " - weight\n", - " - bce_loss: a pytorch nn module\n", - " - encoder: Callable, a function translate name to nhot encoding\n", - " - name_list: a list of entity names\n", - " - to_root_list: an numpy array describe the\n", - " travel distance to root\n", - " - \n", - " \"\"\"\n", - " import torch\n", - " from torch import nn\n", - " to_root_df = pd.DataFrame(\n", - " list(dict(name=name, to_root=node.to_root)\n", - " for name,node in tree.nodes.items()))\n", - " \n", - " to_root_df = to_root_df\\\n", - " .sort_values(by=\"to_root\",ascending=True,)\\\n", - " .reset_index(drop=True)\n", - " \n", - " name_list = list(to_root_df.name)\n", - " n2i = dict((v,k) for k,v in enumerate(name_list))\n", - " to_root_list = np.array(list(to_root_df.to_root))\n", - " \n", - " weight = torch.FloatTensor(calc_weight(to_root_list)*100)\n", - " bce_loss = nn.BCELoss(weight=weight)\n", - " \n", - " eye = np.eye(len(name_list))\n", - " \n", - " def encoder(\n", - " branch: str\n", - " ) -> np.array:\n", - " \"\"\"\n", - " An encoder translate name to nhot encoding\n", - " which we can use as Y label\n", - " \"\"\"\n", - " node = tree[branch]\n", - " idx = np.array(\n", - " list(map(lambda x: n2i[x['name']],\n", - " node.search(tree.root.name))),dtype=int)\n", - " return eye[idx].sum(axis=0)\n", - " \n", - " return dict(weight=weight,\n", - " bce_loss=bce_loss,\n", - " name_list=name_list,\n", - " encoder=encoder,\n", - " to_root_list=to_root_list)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "package = loss_package(tree)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Encoder function" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "from matplotlib import pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "encoder = package['encoder']" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "encoder('Renal Non-Clear Cell Carcinoma')[:200]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Visualize a port of nhot encoding" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAABECAYAAACYhW4wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAHO0lEQVR4nO3dX4hcZx3G8e/jpilUq22atMRk2ySSFnohbQi1oO1NjU2CdquCJAguWAiCBYsIrgSkl1bRC0EsFUOj1Kb4p3QvKmkNYm9smzRuko1pstuY0jXrbpuKLVasqT8v5l04mezMbObPOefF5wPDOefdWc7D75z57TnvzLCKCMzMLD/vqzqAmZl1xw3czCxTbuBmZplyAzczy5QbuJlZptzAzcwy1VMDl7RV0klJ05LG+hXKzMw6U7efA5c0BJwCtgAzwEFgZ0T8uX/xzMyslV6uwG8DpiPidES8C+wDRvoTy8zMOlnWw++uAV4rbM8AH2v3CytXDMW64cs4dfSKHnZbjRs/+g5AltnNyuDXyOC8zd/fiIhVzeO9NHAtMnbRfIykXcAugOvXLOPF/cPc/eFbethtNfbvnwDIMrtZGfwaGZzfxa9eXWy8lymUGWC4sL0WONv8pIh4JCI2R8TmVdcM9bA7MzMr6qWBHwQ2SlovaTmwAxjvTywzM+uk6ymUiDgv6X5gPzAE7ImI431LZmZmbfUyB05EPA083acsZmZ2CfxNTDOzTLmBm5llyg3czCxTbuBmZplyAzczy5QbuJlZptzAzcwy5QZuZpapjg1c0h5J85ImC2MrJD0raSotrx5sTDMza7aUK/BHga1NY2PAgYjYCBxI22ZmVqKODTwingPebBoeAfam9b3AvX3OZWZmHXQ7B35dRMwCpOW1/YtkZmZLMfA3MSXtknRI0qHXz7036N2Zmf3f6LaBz0laDZCW862e6H/oYGY2GN028HFgNK2PAk/1J46ZmS3VUj5G+DjwR+AmSTOS7gO+A2yRNAVsSdtmZlaijv/QISJ2tvjRXX3OYmZml8DfxDQzy5QbuJlZptzAzcwy5QZuZpYpN3Azs0y5gZuZZcoN3MwsU27gZmaZcgM3M8uUG7iZWabcwM3MMqWIKG9n0uvAP4E3Sttp91ZS/5w5ZATn7Dfn7K8cct4QEauaB0tt4ACSDkXE5lJ32oUccuaQEZyz35yzv3LJuRhPoZiZZcoN3MwsU1U08Ecq2Gc3csiZQ0Zwzn5zzv7KJedFSp8DNzOz/vAUiplZpkpr4JK2SjopaVrSWFn77UTSsKTfSzoh6bikr6XxByX9VdJEemyvQdYzko6lPIfS2ApJz0qaSsurK854U6FmE5LekvRAHeopaY+keUmThbFF66eGH6bz9aikTRVm/J6kl1OOJyVdlcbXSfpXoaYPl5GxTc6Wx1jSt1ItT0q6u+KcTxQynpE0kcYrq2fXImLgD2AIeAXYACwHjgA3l7HvJWRbDWxK61cCp4CbgQeBb1SdrynrGWBl09h3gbG0PgY8VHXOpuP+N+CGOtQTuBPYBEx2qh+wHfgtIOB24IUKM34KWJbWHypkXFd8Xg1quegxTq+nI8DlwPrUC4aqytn08+8D3666nt0+yroCvw2YjojTEfEusA8YKWnfbUXEbEQcTutvAyeANdWmuiQjwN60vhe4t8Isze4CXomIV6sOAhARzwFvNg23qt8I8LNoeB64StLqKjJGxDMRcT5tPg+sHXSOTlrUspURYF9E/Dsi/gJM0+gJA9cupyQBXwAeLyPLIJTVwNcArxW2Z6hhk5S0DrgVeCEN3Z9uW/dUPTWRBPCMpJck7Upj10XELDT+GAHXVpbuYju48MVRt3pC6/rV9Zz9Mo07gwXrJf1J0h8k3VFVqILFjnFda3kHMBcRU4WxutWzrbIauBYZq9XHXyR9APg18EBEvAX8GPgIcAswS+NWq2ofj4hNwDbgq5LurDpQK5KWA/cAv0xDdaxnO7U7ZyXtBs4Dj6WhWeD6iLgV+DrwC0kfrCofrY9x7WqZ7OTCC4y61bOjshr4DDBc2F4LnC1p3x1JuoxG834sIn4DEBFzEfFeRPwX+Akl3fK1ExFn03IeeJJGprmFW/u0nK8u4QW2AYcjYg7qWc+kVf1qdc5KGgU+DXwx0oRtmpI4l9ZfojG3fGNVGdsc41rVEkDSMuBzwBMLY3Wr51KU1cAPAhslrU9XZjuA8ZL23VaaB/spcCIiflAYL853fhaYbP7dMkl6v6QrF9ZpvLE1SaOOo+lpo8BT1SS8yAVXN3WrZ0Gr+o0DX0qfRrkd+MfCVEvZJG0FvgncExHvFMZXSRpK6xuAjcDpKjKmDK2O8TiwQ9LlktbTyPli2fmafBJ4OSJmFgbqVs8lKevdUhrv6p+i8Vdtd9Xv3hZyfYLG7dxRYCI9tgM/B46l8XFgdcU5N9B4J/8IcHyhhsA1wAFgKi1X1KCmVwDngA8VxiqvJ40/KLPAf2hcFd7Xqn40bvt/lM7XY8DmCjNO05hDXjg/H07P/Xw6F44Ah4HPVFzLlscY2J1qeRLYVmXONP4o8JWm51ZWz24f/iammVmm/E1MM7NMuYGbmWXKDdzMLFNu4GZmmXIDNzPLlBu4mVmm3MDNzDLlBm5mlqn/AWOmR/i/vBZWAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.imshow(encoder('Renal Non-Clear Cell Carcinoma')[None,:200].repeat(20,axis=0))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### A list of int suggest the travel distance to root of all the categories" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", - " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n", - " 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n", - " 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n", - " 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n", - " 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n", - " 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n", - " 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n", - " 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6])" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "package['to_root_list']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Loss function" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.nn.modules.loss.BCELoss" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bce_loss = package[\"bce_loss\"]\n", - "type(bce_loss)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The loss function as weights according to the structure" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAABxCAYAAAAu7uNXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAI60lEQVR4nO3ca4xcZR3H8e/fbsulIG25pdIGaGyIvDBQGyhiCBFRaAz4ApM2JlSDaaKYiLwwJSYa3qkxhpAYsFEUjRYQUUhTg1ggJiYWyr1cCotUuhZaELmIJlD698V5Bsay2y3bnTknD99Psplznjnp88ucM7+deWankZlIkurygbYDSJKmn+UuSRWy3CWpQpa7JFXIcpekClnuklShgZR7RJwXEVsjYjQi1gxiDknSxGK6/849ImYATwLnAmPAvcDKzHxsWieSJE1oEK/cTwNGM/NvmfkGcANw4QDmkSRNYGQA/+ZxwPa+/THg9L0PiojVwGqAmDnrYwfPPeY9TTLztbcOIKL2yxtvtp1AQ5Z79rQd4cC9j751/xr/ejEzjx7vvkGUe4wz9q5HOzPXAmsBDj12YX545eXvaZL5d780pXDaf7F9Z9sRNGR7/v162xEOWO6u5EXJfvyS+lPe/PeJ7hvEsswYsLBvfwGwYwDzSJImMIhyvxdYHBEnRsQsYAVw2wDmkSRNYNqXZTJzd0R8DbgdmAFcl5mPTvc8kqSJDWLNnczcAGwYxL8tSZqc31CVpApZ7pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekClnuklQhy12SKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVyHKXpApZ7pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalCk5Z7RFwXEbsiYkvf2LyIuCMiniq3c8t4RMTVETEaEQ9HxJJBhpckjW9/Xrn/HDhvr7E1wMbMXAxsLPsA5wOLy89q4JrpiSlJei8mLffM/DPw0l7DFwLXl+3rgc/1jf8iG38F5kTE/OkKK0naP1Ndcz82M58DKLfHlPHjgO19x42VsXeJiNURsTkiNu/+7+tTjCFJGs90f6Aa44zleAdm5trMXJqZS0cOmT3NMSTp/W2q5b6zt9xSbneV8TFgYd9xC4AdU48nSZqKqZb7bcCqsr0KuLVv/OLyVzPLgFd6yzeSpOEZmeyAiFgHnA0cFRFjwHeA7wI3RcQlwLPA58vhG4DlwCjwH+BLA8gsSZrEpOWemSsnuOuccY5N4NIDDSVJOjB+Q1WSKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVyHKXpApZ7pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekClnuklQhy12SKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVyHKXpApZ7pJUIctdkioUmdl2BiLiNWBr2zn24SjgxbZDTMBsU9flfGabui7nm+5sx2fm0ePdMTKNkxyIrZm5tO0QE4mIzV3NZ7ap63I+s01dl/MNM5vLMpJUIctdkirUlXJf23aASXQ5n9mmrsv5zDZ1Xc43tGyd+EBVkjS9uvLKXZI0jSx3SapQ6+UeEedFxNaIGI2INS3Mf11E7IqILX1j8yLijoh4qtzOLeMREVeXrA9HxJIBZ1sYEXdFxOMR8WhEfL1j+Q6OiHsi4qGS78oyfmJEbCr5boyIWWX8oLI/Wu4/YZD5ypwzIuKBiFjfpWwRsS0iHomIByNicxnrxHktc86JiJsj4oly/Z3RhXwRcVJ5zHo/r0bEZV3IVub7RnkubImIdeU50s41l5mt/QAzgKeBRcAs4CHg5CFnOAtYAmzpG/s+sKZsrwG+V7aXA38AAlgGbBpwtvnAkrJ9OPAkcHKH8gVwWNmeCWwq894ErCjj1wJfKdtfBa4t2yuAG4dwfi8Hfg2sL/udyAZsA47aa6wT57XMeT3w5bI9C5jTpXxl3hnA88DxXcgGHAc8AxzSd619sa1rbuAnYJIH4wzg9r79K4ArWshxAv9f7luB+WV7Ps2XrAB+DKwc77gh5bwVOLeL+YBDgfuB02m+gTey9zkGbgfOKNsj5bgYYKYFwEbgk8D68gTvSrZtvLvcO3FegQ+Wkoou5uub59PAX7qSjabctwPzyjW0HvhMW9dc28syvQejZ6yMte3YzHwOoNweU8Zby1vesp1K8+q4M/nKsseDwC7gDpp3Yi9n5u5xMrydr9z/CnDkAONdBXwT2FP2j+xQtgT+GBH3RcTqMtaV87oIeAH4WVnS+klEzO5Qvp4VwLqy3Xq2zPwH8APgWeA5mmvoPlq65tou9xhnrMt/m9lK3og4DPgtcFlmvrqvQ8cZG2i+zHwrM0+heZV8GvCRfWQYWr6I+CywKzPv6x/ex/zDfuzOzMwlwPnApRFx1j6OHXa2EZqlymsy81TgdZqljokM/bor69YXAL+Z7NBxxgZ1zc0FLgROBD4EzKY5vxPNP9BsbZf7GLCwb38BsKOlLP12RsR8gHK7q4wPPW9EzKQp9l9l5i1dy9eTmS8Dd9Osa86JiN7/W9Sf4e185f4jgJcGFOlM4IKI2AbcQLM0c1VHspGZO8rtLuB3NL8Yu3Jex4CxzNxU9m+mKfuu5IOmNO/PzJ1lvwvZPgU8k5kvZOabwC3Ax2npmmu73O8FFpdPk2fRvM26reVM0GRYVbZX0ax198YvLp/ALwNe6b0VHISICOCnwOOZ+cMO5js6IuaU7UNoLu7HgbuAiybI18t9EXBnlgXH6ZaZV2Tmgsw8gea6ujMzv9CFbBExOyIO723TrB1voSPnNTOfB7ZHxEll6Bzgsa7kK1byzpJML0Pb2Z4FlkXEoeW523vc2rnmBv2hx358CLGc5q9Anga+1cL862jWx96k+U16Cc2610bgqXI7rxwbwI9K1keApQPO9gmat2kPAw+Wn+UdyvdR4IGSbwvw7TK+CLgHGKV523xQGT+47I+W+xcN6RyfzTt/LdN6tpLhofLzaO+678p5LXOeAmwu5/b3wNyu5KP58P6fwBF9Y13JdiXwRHk+/BI4qK1rzv9+QJIq1PayjCRpACx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVKH/AfJMaNY9LpEZAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.imshow(bce_loss.weight.numpy()[None,].repeat(200,axis=0))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/55_cosine_search.ipynb b/nbs/55_cosine_search.ipynb index d60644e..db05d32 100644 --- a/nbs/55_cosine_search.ipynb +++ b/nbs/55_cosine_search.ipynb @@ -13,16 +13,6 @@ "metadata": {}, "outputs": [], "source": [ - "# default_exp cosine" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", "import numpy as np\n", "import pandas as pd\n", "from forgebox.category import Category" @@ -37,79 +27,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "# export\n", - "class CosineSearch:\n", - " \"\"\"\n", - " Build a index search on cosine distance\n", - "\n", - " cos = CosineSearch(base_array)\n", - " idx_order = cos(vec)\n", - " \"\"\"\n", - "\n", - " def __init__(self, base):\n", - " assert len(base.shape) == 2,\\\n", - " f\"Base array has to be 2 dimentional, input is {len(base.shape)}\"\n", - " self.base = base\n", - " self.base_norm = self.calc_base_norm(self.base)\n", - " self.normed_base = self.base/self.base_norm[:, None]\n", - " self.dim = self.base.shape[1]\n", - "\n", - " def __len__(self): return base.shape[0]\n", - "\n", - " @staticmethod\n", - " def calc_base_norm(base: np.ndarray) -> np.ndarray:\n", - " return np.sqrt(np.power(base, 2).sum(1))\n", - "\n", - " def search(self, vec: np.ndarray, return_similarity: bool = False):\n", - " if return_similarity:\n", - " similarity = (vec * self.normed_base /\n", - " (np.power(vec, 2).sum())).sum(1)\n", - " order = similarity.argsort()[::-1]\n", - " return order, similarity[order]\n", - " return self(vec)\n", - "\n", - " def __call__(self, vec: np.ndarray) -> np.ndarray:\n", - " \"\"\"\n", - " Return the order index of the closest vector to the furthest\n", - " vec: an 1 dimentional vector\n", - " \"\"\"\n", - " return (vec * self.normed_base).sum(1).argsort()[::-1]\n", - "\n", - "\n", - "class CosineSearchWithCategory(CosineSearch):\n", - " \"\"\"\n", - " Combine with the category manager\n", - " The class can return a dataframe with category information\n", - " \n", - " search_dataframe\n", - " \"\"\"\n", - "\n", - " def __init__(self, base: np.ndarray, category: np.ndarray):\n", - " super().__init__(base)\n", - " self.category = category\n", - " assert len(self.category) >= len(self), \"category number too small\"\n", - "\n", - " def search_dataframe(\n", - " self, vec, return_similarity=True\n", - " ) -> pd.DataFrame:\n", - " \"\"\"\n", - " return a dataframe from the closest\n", - " category to the furthest\n", - " \"\"\"\n", - " if return_similarity:\n", - " idx, similarity = self.search(vec, return_similarity)\n", - " return pd.DataFrame({\n", - " \"category\": self.category.i2c[idx],\n", - " \"idx\": idx,\n", - " \"similarity\": similarity})\n", - " idx = self.search(vec, return_similarity)\n", - " return pd.DataFrame({\n", - " \"category\": self.category.i2c[idx],\n", - " \"idx\": idx})" + "from forgebox.cosine import CosineSearch, CosineSearchWithCategory" ] }, { @@ -121,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -131,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -140,16 +62,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([ 200, 42154, 2439, ..., 45360, 13398, 9083])" + "array([ 200, 45841, 49488, ..., 37457, 31116, 45770])" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -160,18 +82,18 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(array([ 200, 42154, 2439, ..., 45360, 13398, 9083]),\n", - " array([0.22396417, 0.16316632, 0.16174796, ..., 0.06436053, 0.06367614,\n", - " 0.0608117 ]))" + "(array([ 200, 45841, 49488, ..., 37457, 31116, 45770]),\n", + " array([0.22080708, 0.16454399, 0.16373655, ..., 0.07275872, 0.06884953,\n", + " 0.06865599]))" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -184,22 +106,13 @@ "cell_type": "code", "execution_count": 8, "metadata": {}, - "outputs": [], - "source": [ - "# cos_cat = CosineSearchWithCategory(base, Category(list(f\"c{i}\" for i in range(len(base)))))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.21 s, sys: 147 ms, total: 1.36 s\n", - "Wall time: 1.37 s\n" + "CPU times: user 1.37 s, sys: 283 ms, total: 1.66 s\n", + "Wall time: 1.72 s\n" ] } ], @@ -208,11 +121,18 @@ "for i in range(100):\n", " cosine(vec)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/nbs/bert_visualize.ipynb b/nbs/bert_visualize.ipynb deleted file mode 100644 index 32c636c..0000000 --- a/nbs/bert_visualize.ipynb +++ /dev/null @@ -1,315 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Bert Visualize\n", - "> Visualize masked language modeling transformer model" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp bert_visualize" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install transformers" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoModelForMaskedLM,AutoTokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from forgebox.imports import *\n", - "from forgebox.config import Config\n", - "from forgebox.static_file import open_static\n", - "from jinja2 import Template\n", - "from forgebox.html import DOM\n", - "from uuid import uuid4" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = AutoModelForMaskedLM.from_pretrained(\"bert-base-uncased\")\n", - "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\",use_fast=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A piece of sample text" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "text = \"\"\"I must not [MASK].\n", - "Fear is the mind-killer.\n", - "Fear is the little [MASK] that brings total obliteration.\n", - "I will face my fear.\n", - "I will permit it to pass over me and through me.\n", - "And when it has gone past I will turn the inner [MASK] to see its path.\n", - "Where the fear has gone there will be nothing.\n", - "Only I will remain.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class MLMVisualizer:\n", - " def __init__(self,model,tokenizer):\n", - " super().__init__()\n", - " self.model = model\n", - " self.tokenizer = tokenizer\n", - " \n", - " @classmethod\n", - " def from_pretrained(cls,\n", - " tag:\"str, like how you use from_pretrained from transformers\"\n", - " ):\n", - " obj = cls(\n", - " model = AutoModelForMaskedLM.from_pretrained(tag),\n", - " tokenizer = AutoTokenizer.from_pretrained(tag,use_fast=True),\n", - " )\n", - " return obj\n", - " \n", - " def tok(self,text:str,)->[\n", - " torch.FloatTensor,\n", - " torch.BoolTensor,\n", - " list,\n", - " ]:\n", - " \"\"\"\n", - " A specific way of tokenizing.\n", - " with pytorch tensor as input\n", - " with mask tensor specifying where's the [MASK] token\n", - " with offset mapping marking the positions \n", - " in format of list in list\n", - " \"\"\"\n", - " tokenized = self.tokenizer(\n", - " text,\n", - " return_tensors = \"pt\",\n", - " return_offsets_mapping=True\n", - " )\n", - " x = tokenized['input_ids']\n", - " offset_mapping = tokenized['offset_mapping']\n", - " mask = x==self.tokenizer.mask_token_id\n", - " if len(offset_mapping.shape)==3:\n", - " offset_mapping=offset_mapping[0]\n", - " return x,mask,offset_mapping" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vis = MLMVisualizer.from_pretrained(\"bert-base-uncased\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "softmax = nn.Softmax(dim=-1)\n", - "\n", - "def li(x,)->np.array:\n", - " if torch.is_tensor(x):\n", - " x=x.cpu().numpy()\n", - " return x.tolist()\n", - "\n", - "def infer_logits(\n", - " vis,\n", - " y_pred,\n", - " mask) -> Config:\n", - " logits = softmax(y_pred[mask])\n", - " pred_idx = logits.argmax(-1)\n", - " return Config(\n", - " logits=logits,\n", - " pred_idx=pred_idx,\n", - " pred_tokens = vis.tokenizer.convert_ids_to_tokens(pred_idx)\n", - " )\n", - "\n", - "\n", - "MLMVisualizer.infer_logits = infer_logits\n", - "\n", - "def predict_text(\n", - " vis,\n", - " text,\n", - " )->Config:\n", - " with torch.no_grad():\n", - " x,mask,mapper=vis.tok(text)\n", - " y_pred,attention = vis.model(x,output_attentions=True)\n", - " infered = vis.infer_logits(y_pred,mask)\n", - " return Config(\n", - " text = text,\n", - " x = li(x),\n", - " mask = li(mask),\n", - " mapper = li(mapper),\n", - "# y_pred = li(y_pred),\n", - "# logits = li(infered.logits),\n", - " pred_idx=li(infered.pred_idx),\n", - " pred_tokens =infered.pred_tokens,\n", - " attention = list(map(li,attention)),\n", - " )\n", - "MLMVisualizer.predict_text = predict_text\n", - "\n", - "def visualize(vis,\n", - " text):\n", - " result = vis.predict_text(text)\n", - " vis.visualize_result(result)\n", - "\n", - "\n", - "def visualize_result(vis, result: Config):\n", - " template = Template(open_static('mlm/visual.html'))\n", - " js = open_static('mlm/visual.js')\n", - " text = result.text\n", - " delattr(result, 'text')\n", - " output_id = str(uuid4())\n", - " page = template.render(data=json.dumps(result),\n", - " text=text,\n", - " output_id=output_id,\n", - " mlm_visual_js=js)\n", - " DOM(page, \"div\",)()\n", - "\n", - "\n", - "MLMVisualizer.visualize = visualize\n", - "MLMVisualizer.visualize_result = visualize_result" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 606 ms, sys: 25.1 ms, total: 631 ms\n", - "Wall time: 164 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "result = predict_text(vis,text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "vis.visualize(text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Different size of model" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "model = AutoModelForMaskedLM.from_pretrained(\"google/electra-small-generator\")\n", - "tokenizer = AutoTokenizer.from_pretrained(\"google/electra-small-generator\",use_fast=True)\n", - "\n", - "vis = MLMVisualizer(model,tokenizer)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vis.visualize(text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/bilstm-based-search-on-netflix-data.ipynb b/nbs/bilstm-based-search-on-netflix-data.ipynb deleted file mode 100644 index 4fb72f9..0000000 --- a/nbs/bilstm-based-search-on-netflix-data.ipynb +++ /dev/null @@ -1,9087 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# NLP data\n", - ">and all those embedding funs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[Netflix dataset](https://www.kaggle.com/shivamb/netflix-shows) on kaggle\n", - "\n", - "* Create a pytorch dataset for text\n", - "* A BiLSTM model to predict multiple genre\n", - "* Encode the text to vectors using the model we trained\n", - "* Search the closest description" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp data.nlp" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", - "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5" - }, - "outputs": [], - "source": [ - "# export\n", - "import numpy as np # linear algebra\n", - "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", - "import os\n", - "from pathlib import Path\n", - "# Any results you write to the current directory are saved as output." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: forgebox in /Users/salvor/github/forgebox (0.1.1)\r\n", - "Requirement already satisfied: tqdm>=4.25.0 in /Users/salvor/anaconda3/lib/python3.7/site-packages (from forgebox) (4.42.1)\r\n" - ] - } - ], - "source": [ - "!pip install forgebox" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", - "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
show_idtypetitledirectorcastcountrydate_addedrelease_yearratingdurationlisted_indescription
447680145625MovieGod of WarGordon ChanVincent Zhao, Sammo Kam-Bo Hung, Regina Wan, Y...China, Hong KongDecember 16, 20172017NR129 minAction & Adventure, International MoviesA maverick leader and a clever young general t...
432781094074MovieEl Pepe, a Supreme LifeEmir KusturicaJosé Mujica, Emir KusturicaArgentina, Uruguay, SerbiaDecember 27, 20192018TV-1473 minDocumentaries, International MoviesIn this intimate documentary, former Uruguayan...
607880094603TV ShowHighway Thru HellNaNDave Pettitt, Jamie Davis, Adam Gazzola, Kevin...CanadaDecember 3, 20192016TV-PG3 SeasonsReality TVOn the hazardous highways of Canada's interior...
526080044093MovieTeam FoxcatcherJon GreenhalghNaNUnited StatesApril 29, 20162016TV-1491 minDocumentaries, Sports MoviesWith never-before seen home video, this film r...
546481021447MovieThe SilenceJohn R. LeonettiStanley Tucci, Kiernan Shipka, Miranda Otto, K...GermanyApril 10, 20192019TV-1491 minHorror Movies, ThrillersWith the world under attack by deadly creature...
169781043473MovieSGT. Will GardnerMax MartiniMax Martini, Omari Hardwick, Lily Rabe, Elisab...United StatesMay 19, 20192019TV-MA125 minDramasA homeless vet who has PTSD steals a motorcycl...
117880028357MovieLove, RosieChristian DitterLily Collins, Sam Claflin, Christian Cooke, Ja...Germany, United KingdomNovember 20, 20192014R103 minComedies, International Movies, Romantic MoviesOver the years, as they come and go in each ot...
89380190103MovieNaan Sigappu ManithanThiruVishal, Lakshmi Menon, Saranya Ponvannan, Jaya...IndiaOctober 1, 20182014TV-MA147 minAction & Adventure, Dramas, International MoviesAfter his sleeping disorder hinders him from p...
407580156767MovieLa Última FiestaLeandro Mark, Nicolás SilbertNicolás Vázquez, Alan Sabbagh, Benjamín Amadeo...ArgentinaFebruary 1, 20172016TV-MA104 minComedies, International MoviesThree best buddies are thrown into a wild chas...
578380027373TV ShowOh No! It's an Alien InvasionNaNAl Mukadam, Dan Chameroy, Seán Cullen, Stacey ...CanadaMay 31, 20152014TV-Y7-FV2 SeasonsKids' TV, TV Action & Adventure, TV Sci-Fi & F...Nate and his Super Wicked Extreme Emergency Te...
\n", - "
" - ], - "text/plain": [ - " show_id type title \\\n", - "4476 80145625 Movie God of War \n", - "4327 81094074 Movie El Pepe, a Supreme Life \n", - "6078 80094603 TV Show Highway Thru Hell \n", - "5260 80044093 Movie Team Foxcatcher \n", - "5464 81021447 Movie The Silence \n", - "1697 81043473 Movie SGT. Will Gardner \n", - "1178 80028357 Movie Love, Rosie \n", - "893 80190103 Movie Naan Sigappu Manithan \n", - "4075 80156767 Movie La Última Fiesta \n", - "5783 80027373 TV Show Oh No! It's an Alien Invasion \n", - "\n", - " director \\\n", - "4476 Gordon Chan \n", - "4327 Emir Kusturica \n", - "6078 NaN \n", - "5260 Jon Greenhalgh \n", - "5464 John R. Leonetti \n", - "1697 Max Martini \n", - "1178 Christian Ditter \n", - "893 Thiru \n", - "4075 Leandro Mark, Nicolás Silbert \n", - "5783 NaN \n", - "\n", - " cast \\\n", - "4476 Vincent Zhao, Sammo Kam-Bo Hung, Regina Wan, Y... \n", - "4327 José Mujica, Emir Kusturica \n", - "6078 Dave Pettitt, Jamie Davis, Adam Gazzola, Kevin... \n", - "5260 NaN \n", - "5464 Stanley Tucci, Kiernan Shipka, Miranda Otto, K... \n", - "1697 Max Martini, Omari Hardwick, Lily Rabe, Elisab... \n", - "1178 Lily Collins, Sam Claflin, Christian Cooke, Ja... \n", - "893 Vishal, Lakshmi Menon, Saranya Ponvannan, Jaya... \n", - "4075 Nicolás Vázquez, Alan Sabbagh, Benjamín Amadeo... \n", - "5783 Al Mukadam, Dan Chameroy, Seán Cullen, Stacey ... \n", - "\n", - " country date_added release_year rating \\\n", - "4476 China, Hong Kong December 16, 2017 2017 NR \n", - "4327 Argentina, Uruguay, Serbia December 27, 2019 2018 TV-14 \n", - "6078 Canada December 3, 2019 2016 TV-PG \n", - "5260 United States April 29, 2016 2016 TV-14 \n", - "5464 Germany April 10, 2019 2019 TV-14 \n", - "1697 United States May 19, 2019 2019 TV-MA \n", - "1178 Germany, United Kingdom November 20, 2019 2014 R \n", - "893 India October 1, 2018 2014 TV-MA \n", - "4075 Argentina February 1, 2017 2016 TV-MA \n", - "5783 Canada May 31, 2015 2014 TV-Y7-FV \n", - "\n", - " duration listed_in \\\n", - "4476 129 min Action & Adventure, International Movies \n", - "4327 73 min Documentaries, International Movies \n", - "6078 3 Seasons Reality TV \n", - "5260 91 min Documentaries, Sports Movies \n", - "5464 91 min Horror Movies, Thrillers \n", - "1697 125 min Dramas \n", - "1178 103 min Comedies, International Movies, Romantic Movies \n", - "893 147 min Action & Adventure, Dramas, International Movies \n", - "4075 104 min Comedies, International Movies \n", - "5783 2 Seasons Kids' TV, TV Action & Adventure, TV Sci-Fi & F... \n", - "\n", - " description \n", - "4476 A maverick leader and a clever young general t... \n", - "4327 In this intimate documentary, former Uruguayan... \n", - "6078 On the hazardous highways of Canada's interior... \n", - "5260 With never-before seen home video, this film r... \n", - "5464 With the world under attack by deadly creature... \n", - "1697 A homeless vet who has PTSD steals a motorcycl... \n", - "1178 Over the years, as they come and go in each ot... \n", - "893 After his sleeping disorder hinders him from p... \n", - "4075 Three best buddies are thrown into a wild chas... \n", - "5783 Nate and his Super Wicked Extreme Emergency Te... " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "DATA = Path(\"data/netflix_titles.csv\")\n", - "\n", - "df = pd.read_csv(DATA)\n", - "\n", - "df.sample(10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### So... what Y?" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Documentaries 299\n", - "Stand-Up Comedy 273\n", - "Dramas, International Movies 248\n", - "Dramas, Independent Movies, International Movies 186\n", - "Comedies, Dramas, International Movies 174\n", - " ... \n", - "TV Dramas, TV Mysteries, TV Thrillers 1\n", - "Kids' TV, TV Dramas, Teen TV Shows 1\n", - "Romantic TV Shows, Spanish-Language TV Shows, TV Comedies 1\n", - "Classic & Cult TV, TV Horror, TV Mysteries 1\n", - "International TV Shows, Spanish-Language TV Shows, TV Horror 1\n", - "Name: listed_in, Length: 461, dtype: int64" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.listed_in.value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TV-MA 2027\n", - "TV-14 1698\n", - "TV-PG 701\n", - "R 508\n", - "PG-13 286\n", - "NR 218\n", - "PG 184\n", - "TV-Y7 169\n", - "TV-G 149\n", - "TV-Y 143\n", - "TV-Y7-FV 95\n", - "G 37\n", - "UR 7\n", - "NC-17 2\n", - "Name: rating, dtype: int64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.rating.value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "df[\"listed_in\"] = df.listed_in.str\\\n", - ".replace(\"&\",\",\")\\\n", - ".replace(\" , \",\",\")\\\n", - ".replace(\" ,\",\",\")\\\n", - ".replace(\", \",\",\")\\\n", - ".replace(\" , \",\",\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "genre = list(set(i.strip() for i in (\",\".join(list(df.listed_in))).split(\",\")))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total genre: 49\n", - "\n", - "Teen TV Shows\tNature TV\tRomantic TV Shows\tTV Sci-Fi\tAction\tFaith\tTalk Shows\tTV Shows\tTV Dramas\tThrillers\tSci-Fi\tSports Movies\tTV Thrillers\tMovies\tCult Movies\tDocuseries\tTV Comedies\tTV Action\tChildren\tClassic Movies\tKorean TV Shows\tDramas\tTV Horror\tRomantic Movies\tSpirituality\tInternational TV Shows\tIndependent Movies\tStand-Up Comedy\tScience\tHorror Movies\tTV Mysteries\tMusic\tReality TV\tCrime TV Shows\tAdventure\tClassic\tSpanish-Language TV Shows\tFamily Movies\tDocumentaries\tAnime Series\tMusicals\tFantasy\tKids' TV\tAnime Features\tComedies\tBritish TV Shows\tInternational Movies\tLGBTQ Movies\tCult TV\t" - ] - } - ], - "source": [ - "print(f\"Total genre: {len(genre)}\\n\")\n", - "for g in genre:\n", - " print(g,end=\"\\t\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "eye = np.eye(len(genre))\n", - "genre_dict = dict((v,eye[k]) for k,v in enumerate(genre))\n", - "\n", - "def to_nhot(text):\n", - " return np.sum(list(genre_dict[g.strip()] for g in text.split(\",\")),axis=0).astype(np.int)\n", - "\n", - "df[\"genre\"] = df.listed_in.apply(to_nhot)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "PROCESSED = \"processed.csv\"" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "df.to_csv(PROCESSED,index = False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Process the text" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def split_df(df, valid=0.2, ensure_factor=2):\n", - " \"\"\"\n", - " df: dataframe\n", - " valid: valid ratio, default 0.1\n", - " ensure_factor, ensuring the row number to be the multiplication of this factor, default 2\n", - " return train_df, valid_df\n", - " \"\"\"\n", - " split_ = (np.random.rand(len(df)) > valid)\n", - " train_df = df[split_].sample(frac=1.).reset_index().drop(\"index\", axis=1)\n", - " valid_df = df[~split_].sample(frac=1.).reset_index().drop(\"index\", axis=1)\n", - "\n", - " if ensure_factor:\n", - " train_mod = len(train_df) % ensure_factor\n", - " valid_mod = len(valid_df) % ensure_factor\n", - " if train_mod: train_df = train_df[:-train_mod]\n", - " if valid_mod: valid_df = valid_df[:-valid_mod]\n", - " return train_df, valid_df" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "train:5624\tvalid:608\n" - ] - } - ], - "source": [ - "train_df,val_df = split_df(df,valid=0.1)\n", - "print(f\"train:{len(train_df)}\\tvalid:{len(val_df)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "from nltk.tokenize import TweetTokenizer\n", - "tkz = TweetTokenizer()\n", - "def tokenize(txt):\n", - " return tkz.tokenize(txt)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['A', 'man', 'returns', 'home', 'after', 'being', 'released', 'from']" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tokenize(\"A man returns home after being released from \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Generate vocabulary map from material" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - "from itertools import chain\n", - "from multiprocessing import Pool\n", - "from collections import Counter\n", - "from torch.utils.data.dataset import Dataset\n", - "\n", - "class Vocab(object):\n", - " def __init__(self, iterative, tokenize, max_vocab = 20000,nproc=10):\n", - " \"\"\"\n", - " Count the most frequent words\n", - " Make the word<=>index mapping\n", - " \"\"\"\n", - " self.l = list(iterative)\n", - " self.nproc = nproc\n", - " self.max_vocab = max_vocab\n", - " self.tokenize = tokenize\n", - " self.word_beads = self.word_beads_()\n", - " self.counter()\n", - " \n", - " def __len__(self):\n", - " return len(self.words)\n", - " \n", - " def __repr__(self):\n", - " return f\"vocab {self.max_vocab}\"\n", - " \n", - " def word_beads_(self):\n", - " self.p = Pool(self.nproc)\n", - " return list(chain(*list(self.p.map(self.tokenize,self.l))))\n", - " \n", - " def counter(self):\n", - " vals = np.array(list((k,v) for k,v in dict(Counter(self.word_beads)).items()))\n", - " self.words = pd.DataFrame({\"tok\":vals[:,0],\"ct\":vals[:,1]})\n", - " self.words[\"ct\"] = self.words[\"ct\"].apply(int)\n", - " self.words = self.words.sort_values(by= \"ct\",ascending=False)\\\n", - " .reset_index().drop(\"index\",axis=1).head(self.max_vocab-2)\n", - " self.words[\"idx\"] = (np.arange(len(self.words))+2)\n", - " self.words=pd.concat([self.words,pd.DataFrame({\"tok\":[\"\",\"\"],\"ct\":[-1,-1],\"idx\":[0,1]})])\n", - " return self.words\n", - " \n", - " def to_i(self):\n", - " self.t2i = dict(zip(self.words[\"tok\"],self.words[\"idx\"]))\n", - " def to_index(t):\n", - " i = self.t2i.get(t)\n", - " if i==None:\n", - " return 1\n", - " else:\n", - " return i\n", - " return to_index\n", - " \n", - " def to_t(self):\n", - " return np.roll(self.words[\"tok\"],2)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "vocab = Vocab(df.description,tokenize=tokenize)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Vocabulary build from training" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
tokctidx
0four992
1drama993
2brother974
3himself975
4evil956
............
19519Gora119521
19520High-strung119522
19521out-of-the-way119523
0<eos>-10
1<mtk>-11
\n", - "

19524 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " tok ct idx\n", - "0 four 99 2\n", - "1 drama 99 3\n", - "2 brother 97 4\n", - "3 himself 97 5\n", - "4 evil 95 6\n", - "... ... .. ...\n", - "19519 Gora 1 19521\n", - "19520 High-strung 1 19522\n", - "19521 out-of-the-way 1 19523\n", - "0 -1 0\n", - "1 -1 1\n", - "\n", - "[19524 rows x 3 columns]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "vocab.words" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - "\n", - "import torch\n", - "from torch.utils.data.dataloader import DataLoader\n", - "from torch.utils.data._utils.collate import default_collate\n", - "\n", - "class seqData(Dataset):\n", - " def __init__(self,lines,vocab,max_len=-1):\n", - " \"\"\"\n", - " lines: iterative of text, eg. each element a sentence\n", - " vocab:forge.data.nlp.Vocab\n", - " max_len: max length\n", - " \"\"\"\n", - " self.lines = list(lines)\n", - " self.vocab = vocab\n", - " self.to_i = np.vectorize(vocab.to_i())\n", - " self.to_t = vocab.to_t()\n", - " self.bs=1\n", - " self.max_len=max_len\n", - " \n", - " def __len__(self):\n", - " return len(self.lines)\n", - " \n", - " def __getitem__(self,idx):\n", - " \"\"\"\n", - " Translate words to indices\n", - " \"\"\"\n", - " line = self.lines[idx]\n", - " words = self.vocab.tokenize(line)\n", - " if self.max_len>2:\n", - " words = words[:self.max_len-2]\n", - " words = [\"\",]+words+[\"\"]\n", - " return self.to_i(np.array(words))\n", - " \n", - " def backward(self,seq):\n", - " \"\"\"\n", - " This backward has nothing to do with gradrient\n", - " Just to error proof the tokenized line\n", - " \"\"\"\n", - " return \" \".join(self.to_t[seq])\n", - "\n", - " def collate(self,rows):\n", - " \"\"\"\n", - " this collate will pad any sentence that is less then the max length\n", - " \"\"\"\n", - " line_len = torch.LongTensor(list(len(row) for row in rows));\n", - " max_len = line_len.max()\n", - " ones = torch.ones(max_len.item()).long()\n", - " line_pad = max_len-line_len\n", - " return torch.stack(list(torch.cat([torch.LongTensor(row),ones[:pad.item()]]) for row,pad in zip(rows,line_pad[:,None])))\n", - " \n", - "class arrData(Dataset):\n", - " def __init__(self, *arrs):\n", - " self.arr = np.concatenate(arrs,axis=1)\n", - " \n", - " def __len__(self):\n", - " return self.arr.shape[0]\n", - " \n", - " def __getitem__(self,idx):\n", - " return self.arr[idx]\n", - " \n", - " def collate(self,rows):\n", - " return default_collate(rows)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Build vocabulary and train dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "vocab = Vocab(df.description,tokenize=tokenize)\n", - "\n", - "train_seq = seqData(train_df.description,vocab)\n", - "train_y = arrData(np.stack(train_df.genre.values))\n", - "\n", - "val_seq = seqData(val_df.description,vocab)\n", - "val_y = arrData(np.stack(val_df.genre.values))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Size of train dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(5624, 5624)" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(train_seq),len(train_y)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 4908, 1613, 3527, 2113, 4873, 908, 4819, 8186, 569, 8480,\n", - " 8163, 1476, 579, 4989, 1460, 9109, 1544, 8850, 3102, 908, 3233,\n", - " 941, 5001, 1454, 2152, 3246, 5055, 1454, 8480, 4974, 8019, 8019,\n", - " 8589, 579, 0])" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tokenized_line = train_seq[10]\n", - "tokenized_line" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Reconstruct the sentence from indices\n", - "\n", - ">**** means the missing tokens, for they are less frequent than we should hav cared" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\" Comedian Maria Bamford stars in a series inspired by her own life . It's the sometimes surreal story of a woman who loses – and then finds – her s * * t . \"" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "train_seq.backward(tokenized_line)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### A custom made collate function\n", - "\n", - "* Collate function will do the following:\n", - ">Make rows of dataset output into a batch of tensor" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([16, 36])" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "gen = iter(DataLoader(train_seq,batch_size=16, collate_fn=train_seq.collate))\n", - "next(gen).size()" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class fuse(Dataset):\n", - " def __init__(self, *datasets):\n", - " \"\"\"\n", - " A pytorch dataset combining the dataset\n", - " :param datasets:\n", - " \"\"\"\n", - " self.datasets = datasets\n", - " length_s = set(list(len(d) for d in self.datasets))\n", - " assert len(length_s) == 1, \"dataset lenth not matched\"\n", - " self.length = list(length_s)[0]\n", - " self.collates = list(i.collate if hasattr(i,\"collate\") else default_collate for i in datasets)\n", - "\n", - " def __len__(self):\n", - " return self.length\n", - "\n", - " def __getitem__(self, idx):\n", - " return tuple(d.__getitem__(idx) for d in self.datasets)\n", - " \n", - " def collate(self,rows):\n", - " xs = list(zip(*rows))\n", - " return tuple(func(x) for func, x in zip(self.collates,xs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fusing data set" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [], - "source": [ - "train_ds = fuse(train_seq,train_y)\n", - "val_ds = fuse(val_seq,val_y)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Testing Generator" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([16, 36]) torch.Size([16, 49])\n" - ] - } - ], - "source": [ - "gen = iter(DataLoader(train_ds,batch_size=16, collate_fn=train_ds.collate))\n", - "x,y = next(gen)\n", - "print(x.shape,y.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [], - "source": [ - "from torch import nn\n", - "import torch" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [], - "source": [ - "class basicNLP(nn.Module):\n", - " def __init__(self, hs):\n", - " super().__init__()\n", - " self.hs = hs\n", - " self.emb = nn.Embedding(len(vocab),hs)\n", - " self.rnn = nn.LSTM(input_size = hs,hidden_size = hs,batch_first = True)\n", - " self.fc = nn.Sequential(*[\n", - " nn.BatchNorm1d(hs*2),\n", - " nn.ReLU(),\n", - " nn.Linear(hs*2,hs*2),\n", - " nn.BatchNorm1d(hs*2),\n", - " nn.ReLU(),\n", - " nn.Linear(hs*2,49),\n", - " ])\n", - " \n", - " def encoder(self,x):\n", - " x = self.emb(x)\n", - " o1,(h1,c1) = self.rnn(x)\n", - " # run sentence backward\n", - " o2,(h2,c2) = self.rnn(x.flip(dims=[1]))\n", - " return torch.cat([h1[0],h2[0]],dim=1)\n", - " \n", - " def forward(self,x):\n", - " vec = self.encoder(x)\n", - " return self.fc(vec)" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "model = basicNLP(100)" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[ 0, 1477, 8844, 2473, 10957, 10958, 4973, 942, 8502, 8945,\n", - " 940, 8844, 923, 2068, 908, 9196, 957, 2153, 8481, 8851,\n", - " 8873, 1454, 2152, 4741, 1460, 10959, 916, 6700, 579, 0,\n", - " 1, 1, 1, 1, 1, 1],\n", - " [ 0, 9024, 765, 4886, 8086, 948, 8491, 4790, 2467, 2112,\n", - " 1460, 8940, 8833, 8781, 5183, 1118, 3231, 8576, 9070, 12932,\n", - " 579, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1]])" - ] - }, - "execution_count": 71, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x[:2,:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### What does embedding do?" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(torch.Size([16, 36]), torch.Size([16, 36, 100]))" - ] - }, - "execution_count": 72, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x.shape,model.emb(x).shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### What does LSTM return?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For what is LSTM, read this [awesome blog](https://colah.github.io/posts/2015-08-Understanding-LSTMs/), from which I stole the following visualization from" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### In short version\n", - "RNN, it's about sharing model weights throughout temporal sequence, as convolusion share weights in spatial point of view\n", - "https://colah.github.io/posts/2015-08-Understanding-LSTMs/img/RNN-unrolled.png\n", - "* The above green \"A\" areas are shared linear layer\n", - "* GRU & LSTM are advanced version of RNN, with gate control\n", - "* The black arrows above in GRU & LSTM are controlled by gates\n", - "* Gates, are just linear layer with sigmoid activation $\\sigma(x)$, its outputs are between (0,1), hence the name gate, the following illustration is one of the gates in a lstm cell, called input gate\n", - "https://colah.github.io/posts/2015-08-Understanding-LSTMs/img/LSTM3-focus-f.png\n", - "* Other gates control other things like should we forget the early part of then sentence, should we output this .etc" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### In terms of code" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3 µs, sys: 0 ns, total: 3 µs\n", - "Wall time: 7.15 µs\n", - "torch.Size([16, 34, 100])\n", - "torch.Size([1, 16, 100])\n", - "torch.Size([1, 16, 100])\n" - ] - } - ], - "source": [ - "%time \n", - "output,(hidden_state, cell_state) = model.rnn(model.emb(x))\n", - "for t in (output,hidden_state, cell_state):\n", - " print(t.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Disect the iteration through the sentence" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2 µs, sys: 1e+03 ns, total: 3 µs\n", - "Wall time: 6.44 µs\n" - ] - } - ], - "source": [ - "%time\n", - "init_hidden = torch.zeros((1,16,100))\n", - "init_cell = torch.zeros((1,16,100))\n", - "last_h,last_c = init_hidden,init_cell\n", - "outputs = []\n", - "x_vec = model.emb(x)\n", - "for row in range(x.shape[1]):\n", - " last_o, (last_h,last_c) = model.rnn(x_vec[:,row:row+1,:],(last_h,last_c))\n", - " outputs.append(last_o)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "manual_iteration_result = torch.cat(outputs,dim=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([16, 34, 100])" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "manual_iteration_result.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The 2 results are the same, of course, I thought manual python iteration is slower,but they are really close by the above test" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(1.)" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(manual_iteration_result==output).float().mean()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Training" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "lossf = nn.BCEWithLogitsLoss()" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.ftorch.train import Trainer\n", - "from forgebox.ftorch.callbacks import stat\n", - "from forgebox.ftorch.metrics import metric4_bi" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "model = model.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "============================================================\n", - "Notice, The Trainer was not initiated with optimizer\n", - " Use the following syntax to initialize optimizer\n", - " t.opt[\"adm1\"] = torch.optim.Adam(m1.parameters())\n", - " t.opt[\"adg1\"] = torch.optim.Adagrad(m2.parameters())\n", - "============================================================\n", - " \n" - ] - } - ], - "source": [ - "t = Trainer(train_ds, val_dataset=val_ds,batch_size=16,callbacks=[stat], val_callbacks=[stat] ,shuffle=True,)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [], - "source": [ - "t.opt[\"adm1\"] = torch.optim.Adam(model.parameters())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Combined collate function" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "t.train_data.collate_fn = combine_collate(pad_collate,default_collate)\n", - "t.val_data.collate_fn = combine_collate(pad_collate,default_collate)" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [], - "source": [ - "@t.step_train\n", - "def train_step(self):\n", - " self.opt.zero_all()\n", - " x,y = self.data\n", - " y_= model(x)\n", - " loss = lossf(y_,y.float())\n", - " loss.backward()\n", - " self.opt.step_all()\n", - " acc,rec,prec,f1 = metric4_bi(torch.sigmoid(y_),y)\n", - " return dict((k,v.item()) for k,v in zip([\"loss\",\"acc\",\"rec\",\"prec\",\"f1\"],(loss,acc,rec,prec,f1)))\n", - " \n", - "@t.step_val\n", - "def val_step(self):\n", - " x,y = self.data\n", - " y_= model(x)\n", - " loss = lossf(y_,y.float())\n", - " acc,rec,prec,f1 = metric4_bi(torch.sigmoid(y_),y)\n", - " return dict((k,v.item()) for k,v in zip([\"loss\",\"acc\",\"rec\",\"prec\",\"f1\"],(loss,acc,rec,prec,f1)))" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "cea5f378b3864d8386ee467d115c3aa0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1915030.9436050.0183550.252630.0663620.0174.50.012922
min0.1335930.4987240.0000000.000000.0392160.00.00.000000
max0.7176840.9630100.4523811.000000.2105260.0349.04.522615
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.191503 0.943605 0.018355 0.25263 0.066362 0.0 174.5 0.012922\n", - "min 0.133593 0.498724 0.000000 0.00000 0.039216 0.0 0.0 0.000000\n", - "max 0.717684 0.963010 0.452381 1.00000 0.210526 0.0 349.0 4.522615" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3bf5d313f7a644eea01881d432426883", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1661950.9498200.0140630.4552850.0529170.020.00.016862
min0.1336250.9336730.0000000.0000000.0425530.00.00.000000
max0.2083610.9604590.0625001.0000000.1142860.040.00.691359
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.166195 0.949820 0.014063 0.455285 0.052917 0.0 20.0 0.016862\n", - "min 0.133625 0.933673 0.000000 0.000000 0.042553 0.0 0.0 0.000000\n", - "max 0.208361 0.960459 0.062500 1.000000 0.114286 0.0 40.0 0.691359" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3634826c76804e3cbe23fc5341aeb2a6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1609530.9497780.0403620.5036840.0968321.0174.50.013533
min0.1137500.9362240.0000000.0000000.0384621.00.00.000000
max0.2048530.9668370.2285711.0000000.3636361.0349.04.736667
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.160953 0.949778 0.040362 0.503684 0.096832 1.0 174.5 \n", - "min 0.113750 0.936224 0.000000 0.000000 0.038462 1.0 0.0 \n", - "max 0.204853 0.966837 0.228571 1.000000 0.363636 1.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.013533 \n", - "min 0.000000 \n", - "max 4.736667 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "40a061e04cce4838a3cd3b01b51911ae", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1596490.9497880.0567230.4728220.1137871.020.00.013886
min0.1262140.9413260.0000000.0000000.0454551.00.00.000000
max0.1838200.9591840.1621621.0000000.2727271.040.00.569323
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.159649 0.949788 0.056723 0.472822 0.113787 1.0 20.0 0.013886\n", - "min 0.126214 0.941326 0.000000 0.000000 0.045455 1.0 0.0 0.000000\n", - "max 0.183820 0.959184 0.162162 1.000000 0.272727 1.0 40.0 0.569323" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3c3639515f914beba09b0782137f0587", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1502870.9510050.0944980.6192610.16442.0174.50.013062
min0.1074970.9362240.0000000.0000000.04002.00.00.000000
max0.2119890.9668370.2564101.0000000.40002.0349.04.571776
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.150287 0.951005 0.094498 0.619261 0.1644 2.0 174.5 0.013062\n", - "min 0.107497 0.936224 0.000000 0.000000 0.0400 2.0 0.0 0.000000\n", - "max 0.211989 0.966837 0.256410 1.000000 0.4000 2.0 349.0 4.571776" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "10efaef3bbf84d638b328ea476bbea62", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1582330.9501000.0808530.4990420.1519392.020.00.013032
min0.1003520.9387750.0000000.0000000.0416672.00.00.000000
max0.1995640.9630100.2000001.0000000.3333332.040.00.534303
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.158233 0.950100 0.080853 0.499042 0.151939 2.0 20.0 0.013032\n", - "min 0.100352 0.938775 0.000000 0.000000 0.041667 2.0 0.0 0.000000\n", - "max 0.199564 0.963010 0.200000 1.000000 0.333333 2.0 40.0 0.534303" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ddf75d040a4d477194493409f8fd26c4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1390500.9528260.1555670.6474840.2470373.0174.50.013327
min0.1024040.9362240.0000000.0000000.0384623.00.00.000000
max0.1893700.9706630.3823531.0000000.5106383.0349.04.664611
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.139050 0.952826 0.155567 0.647484 0.247037 3.0 174.5 \n", - "min 0.102404 0.936224 0.000000 0.000000 0.038462 3.0 0.0 \n", - "max 0.189370 0.970663 0.382353 1.000000 0.510638 3.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.013327 \n", - "min 0.000000 \n", - "max 4.664611 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3f97e7117429454c9201631d35da6069", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1598430.9496330.1051940.4602750.1730603.020.00.012334
min0.1208510.9375000.0000000.0000000.0408163.00.00.000000
max0.2086860.9642860.2812500.9090910.3921573.040.00.505692
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.159843 0.949633 0.105194 0.460275 0.173060 3.0 20.0 0.012334\n", - "min 0.120851 0.937500 0.000000 0.000000 0.040816 3.0 0.0 0.000000\n", - "max 0.208686 0.964286 0.281250 0.909091 0.392157 3.0 40.0 0.505692" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "023e82e250e54a6c88257def06bc0855", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1256110.9558640.2359540.6935940.3472174.0174.50.012675
min0.0781760.9387750.0652170.3333330.1132084.00.00.000000
max0.1867810.9770410.5161291.0000000.6400004.0349.04.436386
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.125611 0.955864 0.235954 0.693594 0.347217 4.0 174.5 \n", - "min 0.078176 0.938775 0.065217 0.333333 0.113208 4.0 0.0 \n", - "max 0.186781 0.977041 0.516129 1.000000 0.640000 4.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.012675 \n", - "min 0.000000 \n", - "max 4.436386 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "49da2c46c4bd4ff6955500525c50330c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1650550.9477040.1372150.4537380.2065934.020.00.012974
min0.1030900.9323980.0434780.1428570.0701754.00.00.000000
max0.2302520.9591840.2432431.0000000.3600004.040.00.531945
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.165055 0.947704 0.137215 0.453738 0.206593 4.0 20.0 0.012974\n", - "min 0.103090 0.932398 0.043478 0.142857 0.070175 4.0 0.0 0.000000\n", - "max 0.230252 0.959184 0.243243 1.000000 0.360000 4.0 40.0 0.531945" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "640cb7ed83134c11be42092a5b80a9bd", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1106110.9597100.3334780.722260.4509375.0174.50.013399
min0.0729560.9387750.1000000.400000.1600005.00.00.000000
max0.1828410.9770410.6250001.000000.6779665.0349.04.689789
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.110611 0.959710 0.333478 0.72226 0.450937 5.0 174.5 0.013399\n", - "min 0.072956 0.938775 0.100000 0.40000 0.160000 5.0 0.0 0.000000\n", - "max 0.182841 0.977041 0.625000 1.00000 0.677966 5.0 349.0 4.689789" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "dc5cbd8b675d4408b48a8de5f87bfcdb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1758970.9464290.1524440.3911540.2216505.020.00.012744
min0.1218430.9349490.0000000.0000000.0392165.00.00.000000
max0.2248490.9604590.3428570.7500000.4210535.040.00.522507
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.175897 0.946429 0.152444 0.391154 0.221650 5.0 20.0 0.012744\n", - "min 0.121843 0.934949 0.000000 0.000000 0.039216 5.0 0.0 0.000000\n", - "max 0.224849 0.960459 0.342857 0.750000 0.421053 5.0 40.0 0.522507" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "98ae22e34c8147148e35b8c51c547de5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.0963000.9639710.4302030.7565480.5434976.0174.50.012643
min0.0604330.9413260.1707320.4736840.2641516.00.00.000000
max0.1810050.9808670.7272731.0000000.7804886.0349.04.425211
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.096300 0.963971 0.430203 0.756548 0.543497 6.0 174.5 \n", - "min 0.060433 0.941326 0.170732 0.473684 0.264151 6.0 0.0 \n", - "max 0.181005 0.980867 0.727273 1.000000 0.780488 6.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.012643 \n", - "min 0.000000 \n", - "max 4.425211 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "076eaceb7e9545fe9a3ee98d6cfcfe8c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.1855050.9465840.1858380.4306420.2561446.020.00.012818
min0.1169730.9272960.0540540.1250000.0754726.00.00.000000
max0.2595560.9642860.3939390.6923080.4864866.040.00.525531
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.185505 0.946584 0.185838 0.430642 0.256144 6.0 20.0 0.012818\n", - "min 0.116973 0.927296 0.054054 0.125000 0.075472 6.0 0.0 0.000000\n", - "max 0.259556 0.964286 0.393939 0.692308 0.486486 6.0 40.0 0.525531" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8349300bfc7a4b7ebe4f4b4d1cce5972", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.0830810.9677270.5079440.7805870.6111337.0174.50.014348
min0.0525580.9426020.2380950.4848480.3278697.00.00.000000
max0.1404120.9846940.7777781.0000000.8235297.0349.05.021668
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.083081 0.967727 0.507944 0.780587 0.611133 7.0 174.5 \n", - "min 0.052558 0.942602 0.238095 0.484848 0.327869 7.0 0.0 \n", - "max 0.140412 0.984694 0.777778 1.000000 0.823529 7.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.014348 \n", - "min 0.000000 \n", - "max 5.021668 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f976fd0613eb4a3e8efc972fcec07f16", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.2045990.9451530.1965500.4180750.2638337.020.00.012317
min0.1322900.9247450.0212770.0714290.0327877.00.00.000000
max0.3910810.9642860.3333331.0000000.4400007.040.00.504995
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.204599 0.945153 0.196550 0.418075 0.263833 7.0 20.0 0.012317\n", - "min 0.132290 0.924745 0.021277 0.071429 0.032787 7.0 0.0 0.000000\n", - "max 0.391081 0.964286 0.333333 1.000000 0.440000 7.0 40.0 0.504995" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b9fa56c535a04bc3950feb4bfd1c5683", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.0723080.9719040.5822420.8115310.6743358.0174.50.013954
min0.0444030.9515310.3428570.5454550.4210538.00.00.000000
max0.1131980.9897960.8750001.0000000.8974368.0349.04.884068
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.072308 0.971904 0.582242 0.811531 0.674335 8.0 174.5 \n", - "min 0.044403 0.951531 0.342857 0.545455 0.421053 8.0 0.0 \n", - "max 0.113198 0.989796 0.875000 1.000000 0.897436 8.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.013954 \n", - "min 0.000000 \n", - "max 4.884068 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e1bdc8b646984fb188b48c1246a37290", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.2194910.9416690.1858540.3450280.2442438.020.00.015648
min0.1380230.9272960.0000000.0000000.0833338.00.00.000000
max0.3261870.9591840.3414630.6000000.4137938.040.00.641576
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.219491 0.941669 0.185854 0.345028 0.244243 8.0 20.0 0.015648\n", - "min 0.138023 0.927296 0.000000 0.000000 0.083333 8.0 0.0 0.000000\n", - "max 0.326187 0.959184 0.341463 0.600000 0.413793 8.0 40.0 0.641576" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9309f80e873f47f5a4972fc7e821b227", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=350.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.0621070.9758990.6560510.8350320.7311279.0174.50.013415
min0.0314620.9579080.3902440.6296300.5000009.00.00.000000
max0.1034140.9923470.9333331.0000000.9142869.0349.04.695169
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter \\\n", - "mean 0.062107 0.975899 0.656051 0.835032 0.731127 9.0 174.5 \n", - "min 0.031462 0.957908 0.390244 0.629630 0.500000 9.0 0.0 \n", - "max 0.103414 0.992347 0.933333 1.000000 0.914286 9.0 349.0 \n", - "\n", - " timestamp \n", - "mean 0.013415 \n", - "min 0.000000 \n", - "max 4.695169 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a2180592d3e949008ad9b4f26446ecd8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=41.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
lossaccrecprecf1epochitertimestamp
mean0.2329480.9424150.1912210.3429910.2494189.020.00.014030
min0.1394470.9260200.0000000.0000000.0769239.00.00.000000
max0.3202010.9693880.4117650.6190480.4642869.040.00.575232
\n", - "
" - ], - "text/plain": [ - " loss acc rec prec f1 epoch iter timestamp\n", - "mean 0.232948 0.942415 0.191221 0.342991 0.249418 9.0 20.0 0.014030\n", - "min 0.139447 0.926020 0.000000 0.000000 0.076923 9.0 0.0 0.000000\n", - "max 0.320201 0.969388 0.411765 0.619048 0.464286 9.0 40.0 0.575232" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "t.train(10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Search similar" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "model = model.eval()\n", - "dl = DataLoader(train_seq, batch_size=32, collate_fn=pad_collate)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "text_gen = iter(dl)\n", - "result = []\n", - "for i in range(len(dl)):\n", - " x=next(text_gen)\n", - " x = x.cuda()\n", - " x_vec = model.encoder(x)\n", - " result.append(x_vec.cpu())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A vector representing each of the sentence" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(5590, 200)" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "result_vec = torch.cat(result,dim=0).detach().numpy()\n", - "result_vec.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "def to_idx(line):\n", - " words = train_seq.vocab.tokenize(line)\n", - " words = [\"\",]+words+[\"\"]\n", - " return train_seq.to_i(np.array(words))[None,:]" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[ 0, 913, 0]]),\n", - " array([[ 0, 2153, 5056, 1442, 1438, 2153, 5056, 0]]))" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "to_idx(\"this\"), to_idx(\"to be or not to be\")" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [], - "source": [ - "def to_vec(line):\n", - " vec = torch.LongTensor(to_idx(line)).cuda()\n", - " return model.encoder(vec).cpu().detach().numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[-0.2864027 , -0.1188629 , -0.02617935, -0.24850701, -0.20578709,\n", - " 0.06889898, 0.05878495, -0.22658055, 0.15024377, -0.31303164,\n", - " 0.4958601 , -0.00204021, 0.17621423, -0.2538225 , -0.3451157 ,\n", - " -0.23131247, -0.06265341, -0.17155428, -0.00899762, -0.2577241 ,\n", - " -0.02896317, -0.4555603 , 0.6856887 , -0.70418745, 0.11082602,\n", - " -0.23981036, -0.21201135, -0.43933266, -0.40148616, -0.48364577,\n", - " 0.42605698, 0.41181046, -0.14798354, 0.05320957, -0.4300459 ,\n", - " -0.06580015, 0.01534137, 0.02928652, -0.53414524, -0.02051809,\n", - " -0.47986796, -0.12036817, -0.00229292, 0.2772647 , 0.1102341 ,\n", - " -0.40527657, -0.11229473, -0.42787483, 0.40304342, 0.3992268 ,\n", - " 0.1696693 , -0.6523132 , -0.3679182 , 0.0087082 , -0.01391423,\n", - " -0.21790238, -0.3263417 , -0.29073295, 0.45376575, -0.02547877,\n", - " -0.08805989, -0.04059793, 0.32122698, -0.10253229, 0.29216015,\n", - " -0.29081804, 0.7000031 , -0.07116397, -0.03899158, 0.00963022,\n", - " -0.00503215, -0.18202703, -0.06382827, 0.04819749, -0.39615613,\n", - " -0.06416079, 0.11534453, 0.05590353, 0.18084338, 0.12755607,\n", - " -0.40813434, -0.08283492, -0.48865405, -0.0570746 , -0.20188878,\n", - " -0.49008507, -0.27526015, -0.3434001 , -0.24542893, 0.01706186,\n", - " 0.3014823 , -0.00861376, 0.43119976, 0.11287364, 0.10990695,\n", - " -0.05565942, 0.3471237 , 0.39106023, 0.12372674, -0.1318522 ,\n", - " -0.2864027 , -0.1188629 , -0.02617935, -0.24850701, -0.20578709,\n", - " 0.06889898, 0.05878495, -0.22658055, 0.15024377, -0.31303164,\n", - " 0.4958601 , -0.00204021, 0.17621423, -0.2538225 , -0.3451157 ,\n", - " -0.23131247, -0.06265341, -0.17155428, -0.00899762, -0.2577241 ,\n", - " -0.02896317, -0.4555603 , 0.6856887 , -0.70418745, 0.11082602,\n", - " -0.23981036, -0.21201135, -0.43933266, -0.40148616, -0.48364577,\n", - " 0.42605698, 0.41181046, -0.14798354, 0.05320957, -0.4300459 ,\n", - " -0.06580015, 0.01534137, 0.02928652, -0.53414524, -0.02051809,\n", - " -0.47986796, -0.12036817, -0.00229292, 0.2772647 , 0.1102341 ,\n", - " -0.40527657, -0.11229473, -0.42787483, 0.40304342, 0.3992268 ,\n", - " 0.1696693 , -0.6523132 , -0.3679182 , 0.0087082 , -0.01391423,\n", - " -0.21790238, -0.3263417 , -0.29073295, 0.45376575, -0.02547877,\n", - " -0.08805989, -0.04059793, 0.32122698, -0.10253229, 0.29216015,\n", - " -0.29081804, 0.7000031 , -0.07116397, -0.03899158, 0.00963022,\n", - " -0.00503215, -0.18202703, -0.06382827, 0.04819749, -0.39615613,\n", - " -0.06416079, 0.11534453, 0.05590353, 0.18084338, 0.12755607,\n", - " -0.40813434, -0.08283492, -0.48865405, -0.0570746 , -0.20188878,\n", - " -0.49008507, -0.27526015, -0.3434001 , -0.24542893, 0.01706186,\n", - " 0.3014823 , -0.00861376, 0.43119976, 0.11287364, 0.10990695,\n", - " -0.05565942, 0.3471237 , 0.39106023, 0.12372674, -0.1318522 ]],\n", - " dtype=float32),\n", - " array([[-4.67454530e-02, -9.31215361e-02, 1.03428781e-01,\n", - " -7.04722330e-02, -1.75118193e-01, 2.06147537e-01,\n", - " -1.80195719e-01, -2.96070606e-01, -6.63760230e-02,\n", - " -1.17778860e-01, 4.16386664e-01, -2.82360259e-02,\n", - " -9.79896635e-02, -1.64554998e-01, -4.22353059e-01,\n", - " -2.57056504e-01, 9.54333842e-02, -4.04358506e-01,\n", - " -2.89067421e-02, 2.24286512e-01, -2.91498333e-01,\n", - " -5.15985727e-01, 6.25436842e-01, -3.82335573e-01,\n", - " 9.66877778e-05, -5.79554796e-01, 2.49115989e-01,\n", - " -1.89282358e-01, -5.12709558e-01, -3.26256692e-01,\n", - " -4.69405144e-01, -7.64686316e-02, 1.47889346e-01,\n", - " 4.15703245e-02, -2.66019195e-01, -2.08178125e-02,\n", - " 2.30910387e-02, 4.07335609e-01, -3.15426886e-01,\n", - " -2.41327912e-01, -4.37703043e-01, 1.69361576e-01,\n", - " -5.28552115e-01, 3.17790836e-01, -2.26195082e-01,\n", - " 1.98947728e-01, -5.99721745e-02, -1.17049702e-01,\n", - " 3.48403037e-01, 1.16154104e-01, -9.79769006e-02,\n", - " -5.31215489e-01, -3.45111728e-01, -1.51543558e-01,\n", - " -4.74260300e-02, -3.10880184e-01, -2.34046459e-01,\n", - " -4.89330530e-01, 1.92080900e-01, 8.46387047e-05,\n", - " 5.20091116e-01, 1.19925300e-02, -2.24701278e-02,\n", - " 3.68952841e-01, 3.63212615e-01, -1.76386997e-01,\n", - " 8.03452671e-01, -1.92376554e-01, 2.48082243e-02,\n", - " 4.32565331e-01, -3.37531790e-02, -4.21101004e-02,\n", - " -2.10536309e-02, 1.79137215e-02, -5.51163614e-01,\n", - " 1.79473273e-02, -2.56144851e-01, 1.43536568e-01,\n", - " 1.14509970e-01, -4.08435524e-01, -6.10258477e-03,\n", - " -1.82999484e-02, -3.36512983e-01, -4.28514257e-02,\n", - " -5.26481271e-01, -4.67332840e-01, -5.00806391e-01,\n", - " -1.97101533e-01, -3.14959913e-01, 2.88293674e-03,\n", - " -7.90096354e-03, -1.98622406e-01, 3.48969668e-01,\n", - " -2.70279735e-01, -1.35412514e-01, -1.51752485e-02,\n", - " -6.74099028e-02, -9.12430584e-02, 6.52649999e-02,\n", - " -4.96096015e-02, -8.09527710e-02, -1.94839135e-01,\n", - " 7.93975685e-03, 2.51075149e-01, -1.21566914e-01,\n", - " 1.87434196e-01, -4.09827203e-01, -1.85767949e-01,\n", - " -4.50579636e-02, -8.25710967e-03, 4.34691131e-01,\n", - " -1.74075253e-02, -2.26057798e-01, -9.93655398e-02,\n", - " -4.55289811e-01, -1.77727148e-01, 1.51167855e-01,\n", - " -3.92441899e-01, -3.76366675e-02, 1.91804245e-01,\n", - " -2.46650815e-01, -2.66600817e-01, 5.98052680e-01,\n", - " -5.53917885e-01, -5.27672432e-02, -4.28468704e-01,\n", - " 3.21030438e-01, -5.24240211e-02, -5.80894887e-01,\n", - " -2.62316704e-01, -5.54102123e-01, -2.63558865e-01,\n", - " 7.92699903e-02, 3.08560506e-02, -6.34330451e-01,\n", - " -2.22378112e-02, 1.90087445e-02, 3.06508392e-01,\n", - " -4.17445600e-01, -1.65813014e-01, -3.87870014e-01,\n", - " 2.00837955e-01, -5.82205415e-01, 2.10029915e-01,\n", - " -1.73551142e-01, 2.62706548e-01, -4.51332517e-02,\n", - " -2.98126251e-01, 1.22165889e-01, 2.69887537e-01,\n", - " -9.97079089e-02, -4.44641918e-01, -1.94629923e-01,\n", - " -9.44814011e-02, -5.58075830e-02, -3.08131754e-01,\n", - " -1.35656327e-01, -4.49017793e-01, 1.57232210e-01,\n", - " 6.85075764e-03, 1.95502162e-01, -3.58620263e-03,\n", - " -7.33885169e-02, 2.54277617e-01, 3.10389578e-01,\n", - " -2.30608821e-01, 7.85782695e-01, -1.99842900e-01,\n", - " 2.65866872e-02, 4.02122647e-01, -4.90787327e-02,\n", - " 2.89043874e-01, 7.39292949e-02, 2.30548643e-02,\n", - " -5.23956776e-01, 8.47117510e-03, -2.22430736e-01,\n", - " 2.78976131e-02, 1.98936775e-01, -3.27527881e-01,\n", - " -3.42491508e-01, 9.65497643e-03, -3.23785730e-02,\n", - " -6.30644411e-02, -4.23567891e-01, -4.62936759e-01,\n", - " -5.00409305e-01, -2.56481886e-01, -2.42299706e-01,\n", - " 6.78150402e-03, -1.47003448e-03, -8.91473442e-02,\n", - " 3.01649749e-01, -2.08600447e-01, -2.30947211e-01,\n", - " 1.51244365e-02, 9.57912132e-02, -7.86154643e-02,\n", - " 9.30957273e-02, -1.24067500e-01]], dtype=float32))" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "to_vec(\"this\"), to_vec(\"to be or not to be\")" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [], - "source": [ - "def l2norm(x):\n", - " \"\"\"\n", - " L2 Norm\n", - " \"\"\"\n", - " return np.linalg.norm(x,2,1).reshape(-1,1)" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [], - "source": [ - "pd.set_option(\"max_colwidth\",150)\n", - "\n", - "def search(line):\n", - " vec = to_vec(line)\n", - " sim = ((vec* result_vec)/l2norm(result_vec)).sum(-1)\n", - " return pd.DataFrame({\"text\":train_seq.lines,\"sim\":sim})\\\n", - " .sort_values(by=\"sim\",ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
textsim
4699The year is 2041 and a dispute between a man and his wife has set the human race back. In this battle of the sexes, the primitive life isn't so si...2.796010
1184Ya-nuo's been raised as a boy. Now at age 25, she's caught the eye of a triad leader's sister. But what happens when she reveals her true gender?2.747936
47Scouted by a famous Spanish club, Valt Aoi heads to Spain. With their sights on the World League, he and his teammates face the European League fi...2.684218
1116Led by seventh-grader C.J., three students who have been warned about the dangers of high school decide to make the best of their middle-school ye...2.633265
5380Love is in the air as Zoe and friends go on a quest to find a fabled Maid's Stone. But when rivalry blinds them to danger, it's Raven to the rescue!2.550404
4741The whole huggable gang is back, bringing tales of caring and sharing to a new generation. And now the Care Bear Cousins are here to join the fun!2.546750
4609Captain Atomic – once a superhero, now a sock puppet – can only activate his powers with the help of Joey, his new partner and biggest fan.2.545124
3574Sometimes being shady is the only way to survive, a fact these sneaky animal \"hustlers\" – including orcas, owls and otters – use to their advantage.2.517003
2686Comedian Maria Bamford stars in a series inspired by her own life. It's the sometimes surreal story of a woman who loses – and then finds – her s**t.2.510907
1039Thom tells her grandson about his grandfather, a sailor who left 61 years ago to seek his fortune, and asks him to find out what happened to her s...2.499740
\n", - "
" - ], - "text/plain": [ - " text \\\n", - "4699 The year is 2041 and a dispute between a man and his wife has set the human race back. In this battle of the sexes, the primitive life isn't so si... \n", - "1184 Ya-nuo's been raised as a boy. Now at age 25, she's caught the eye of a triad leader's sister. But what happens when she reveals her true gender? \n", - "47 Scouted by a famous Spanish club, Valt Aoi heads to Spain. With their sights on the World League, he and his teammates face the European League fi... \n", - "1116 Led by seventh-grader C.J., three students who have been warned about the dangers of high school decide to make the best of their middle-school ye... \n", - "5380 Love is in the air as Zoe and friends go on a quest to find a fabled Maid's Stone. But when rivalry blinds them to danger, it's Raven to the rescue! \n", - "4741 The whole huggable gang is back, bringing tales of caring and sharing to a new generation. And now the Care Bear Cousins are here to join the fun! \n", - "4609 Captain Atomic – once a superhero, now a sock puppet – can only activate his powers with the help of Joey, his new partner and biggest fan. \n", - "3574 Sometimes being shady is the only way to survive, a fact these sneaky animal \"hustlers\" – including orcas, owls and otters – use to their advantage. \n", - "2686 Comedian Maria Bamford stars in a series inspired by her own life. It's the sometimes surreal story of a woman who loses – and then finds – her s**t. \n", - "1039 Thom tells her grandson about his grandfather, a sailor who left 61 years ago to seek his fortune, and asks him to find out what happened to her s... \n", - "\n", - " sim \n", - "4699 2.796010 \n", - "1184 2.747936 \n", - "47 2.684218 \n", - "1116 2.633265 \n", - "5380 2.550404 \n", - "4741 2.546750 \n", - "4609 2.545124 \n", - "3574 2.517003 \n", - "2686 2.510907 \n", - "1039 2.499740 " - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "search(\"Experience our planet's natural beauty\").head(10)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
textsim
1588To her Indian parents' dismay, London-born Jasmeet \"Jazz\" Malhotra longs for everything Western, including her British boyfriend. On a family trip...2.468083
4285Quick to throw punches in the name of justice, a young man must find a calmer way to win over the pacifist father of the girl he wishes to marry.2.414226
504When Parisian Elsa gets hung up on her ex, her best friends secretly hire a male escort to help her move on. But their plan works a little too well.2.391777
2595Motu and Patlu want to help a circus lion get back to the jungle. On the way, the three become caught up in an exciting adventure in the forest.2.369020
652Hoping to find a magical root, a monster has captured farmers in the land of Vyom. It’s up to Bheem and the gang to foil his plan and save the kin...2.363151
2042Tired of her passionless marriage, Marianne wants a separation from her husband, Gustav, who, in response, decides to make a big change of his own.2.361425
1668In 1962 Brooklyn, a Puerto Rican teen who joins a gang is seduced by violence and heroin. But can his mother's love and faith in God save him?2.344456
2478Maya finally hooks up with her online dream girl, only to discover she’s deeply involved with an older sugar daddy – a man Maya knows all too well.2.337689
1324After an argument with her dad, a young woman from a family of macho truck drivers is kicked out of the home and must make her own success as a tr...2.334984
177Little Singham is in London to meet the queen, but when the famed Kohinoor Diamond gets stolen, the kid cop goes on a wild, citywide hunt for the ...2.310921
\n", - "
" - ], - "text/plain": [ - " text \\\n", - "1588 To her Indian parents' dismay, London-born Jasmeet \"Jazz\" Malhotra longs for everything Western, including her British boyfriend. On a family trip... \n", - "4285 Quick to throw punches in the name of justice, a young man must find a calmer way to win over the pacifist father of the girl he wishes to marry. \n", - "504 When Parisian Elsa gets hung up on her ex, her best friends secretly hire a male escort to help her move on. But their plan works a little too well. \n", - "2595 Motu and Patlu want to help a circus lion get back to the jungle. On the way, the three become caught up in an exciting adventure in the forest. \n", - "652 Hoping to find a magical root, a monster has captured farmers in the land of Vyom. It’s up to Bheem and the gang to foil his plan and save the kin... \n", - "2042 Tired of her passionless marriage, Marianne wants a separation from her husband, Gustav, who, in response, decides to make a big change of his own. \n", - "1668 In 1962 Brooklyn, a Puerto Rican teen who joins a gang is seduced by violence and heroin. But can his mother's love and faith in God save him? \n", - "2478 Maya finally hooks up with her online dream girl, only to discover she’s deeply involved with an older sugar daddy – a man Maya knows all too well. \n", - "1324 After an argument with her dad, a young woman from a family of macho truck drivers is kicked out of the home and must make her own success as a tr... \n", - "177 Little Singham is in London to meet the queen, but when the famed Kohinoor Diamond gets stolen, the kid cop goes on a wild, citywide hunt for the ... \n", - "\n", - " sim \n", - "1588 2.468083 \n", - "4285 2.414226 \n", - "504 2.391777 \n", - "2595 2.369020 \n", - "652 2.363151 \n", - "2042 2.361425 \n", - "1668 2.344456 \n", - "2478 2.337689 \n", - "1324 2.334984 \n", - "177 2.310921 " - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "search(\"love story,marriage, girl\").head(10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Well, usually it should be more accurate if we have more data" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "state": { - "023e82e250e54a6c88257def06bc0855": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c707a9347c1747fd894259555ca3b828", - "IPY_MODEL_e8f95ab5b4fe4268abf6ca23405ba3c7" - ], - "layout": "IPY_MODEL_461e427a32fa464894beb3e0afaab32b" - } - }, - "039a7ab348934d0588ad3963dcf0f537": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "04eb9ac2e5404045a03de405a43ef519": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "0612263531e14be192de9ee979e0f462": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "06442feaea2140eba8041e0a30fe1e80": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "076eaceb7e9545fe9a3ee98d6cfcfe8c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_2c6e13c8066d461b9e9377081abfc40b", - "IPY_MODEL_d0e4d29a658a47a1a8ec581f41deb332" - ], - "layout": "IPY_MODEL_b965434228194f3c89c8c09d0a06f935" - } - }, - "08ad0b01dbba4803abf9c2bef2d2b596": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8501776035de4302bfe402bad761204e", - "placeholder": "​", - "style": "IPY_MODEL_c5c8dd3bba1e4ec3940c2217fc654848", - "value": " 41/41 [00:03<00:00, 11.41it/s, loss=0.165, acc=0.948, rec=0.137, prec=0.454, f1=0.207]" - } - }, - "0a782acb027a4d68b77897679075f0ee": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5cb237cfc9714d85ae6454b540111757", - "placeholder": "​", - "style": "IPY_MODEL_bdfaf13535b94bb8af718c267941dc9b", - "value": " 41/41 [00:00<00:00, 69.30it/s, loss=0.158, acc=0.95, rec=0.0809, prec=0.499, f1=0.152]" - } - }, - "0e04aab9e5f64f9a92a2cb94725e11c2": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "10efaef3bbf84d638b328ea476bbea62": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_da15936127e144a29ca03dde03a2a8f5", - "IPY_MODEL_0a782acb027a4d68b77897679075f0ee" - ], - "layout": "IPY_MODEL_b04c40ef793c48cf8c631fa22e498c61" - } - }, - "126fb88c8f004f749299b92327f6b80b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "14a8421f71824320bd68a112e5a7aac0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "156c73fe0e64416dbc89c10fe3ccb8b3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "16787b498eee416dbcbdc9da029bc4b4": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1681323fbd124c5b9f0b69dc86a3c01d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_86f9885066e54c80be3e8f96c01af274", - "placeholder": "​", - "style": "IPY_MODEL_f20173f038c840f78d9329b78872b472", - "value": " 350/350 [00:17<00:00, 19.57it/s, loss=0.0847, acc=0.967, rec=0.511, prec=0.756, f1=0.608]" - } - }, - "1c01671c50d2417483279004a9a979b3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "200ff177339947fd9bd012f1698c1acb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "20c48c3450914d5d971a179de0a14d9d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "27b787ef73a04322aa773003481e2270": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_83084bf428ac435db310fbb5c20c9568", - "placeholder": "​", - "style": "IPY_MODEL_474de7c7a58e4afabdb040ac600aebc0", - "value": " 350/350 [00:11<00:00, 29.78it/s, loss=0.0757, acc=0.971, rec=0.575, prec=0.777, f1=0.657]" - } - }, - "28ea9ede92954917933c0a04b3a041f9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2c6e13c8066d461b9e9377081abfc40b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_6_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_3c708db4ab8542e982c59c668c1725ae", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_14a8421f71824320bd68a112e5a7aac0", - "value": 41 - } - }, - "2dedf80d90314a76a9ce08ea848687a3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3634826c76804e3cbe23fc5341aeb2a6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_fe8b438ac1504c84baa784656b934d48", - "IPY_MODEL_82eb8ce99014431ba74309a168d0b0c9" - ], - "layout": "IPY_MODEL_8f8b33efd16a4059910e9b945e885451" - } - }, - "3a6ac8bbac6c42928b140c731b7c3131": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_43c415ec75e64d8e9530a3cbfcebc4fe", - "placeholder": "​", - "style": "IPY_MODEL_b9190069ef54401faae76f05a24a1699", - "value": " 350/350 [00:29<00:00, 11.97it/s, loss=0.111, acc=0.961, rec=0.356, prec=0.744, f1=0.478]" - } - }, - "3bf5d313f7a644eea01881d432426883": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_de17fc424fdb47fe844224df94c5b939", - "IPY_MODEL_8818291bf0bb46d28532173511181808" - ], - "layout": "IPY_MODEL_5056faf1de6c4adb9ed502663996d03c" - } - }, - "3bfebf4c1fd24fb3addd88a828219197": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "3c3639515f914beba09b0782137f0587": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6faf56957e5c4d9586d0d688b9dc12f0", - "IPY_MODEL_57d9a09fbf34412d94bd3e33e8047721" - ], - "layout": "IPY_MODEL_4c129552009445578778dbddaadd6260" - } - }, - "3c708db4ab8542e982c59c668c1725ae": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3d3a021474bc4d8ea62899e335576b4d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3f97e7117429454c9201631d35da6069": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_636c48598d964ebf8936f9c4c3116195", - "IPY_MODEL_da248207c8d8415abb0506ff97b949e8" - ], - "layout": "IPY_MODEL_28ea9ede92954917933c0a04b3a041f9" - } - }, - "40a061e04cce4838a3cd3b01b51911ae": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_ff6dfefcf7aa4033a8df49f42a1ecab3", - "IPY_MODEL_8d98c0c989ff491293573248bb8af7dd" - ], - "layout": "IPY_MODEL_16787b498eee416dbcbdc9da029bc4b4" - } - }, - "43c415ec75e64d8e9530a3cbfcebc4fe": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "445f5fcac5154a66889a0ec782416001": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "45b5e2cd4f3d4f8b8eb673307cff78d3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "461e427a32fa464894beb3e0afaab32b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "474de7c7a58e4afabdb040ac600aebc0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "48ea76e093c346f2a65c7f667d7635eb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_7_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_a8b2117b6c1e4a4e98c4375ab25afdac", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_04eb9ac2e5404045a03de405a43ef519", - "value": 41 - } - }, - "49da2c46c4bd4ff6955500525c50330c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c463c3b9fc774dbc9cc7a311eb297fe9", - "IPY_MODEL_08ad0b01dbba4803abf9c2bef2d2b596" - ], - "layout": "IPY_MODEL_200ff177339947fd9bd012f1698c1acb" - } - }, - "4a801477f8ef4a87be58eb69291f6f75": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4c129552009445578778dbddaadd6260": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4dc1dbf7e51b4e2a8d3c1f99a4966be4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_126fb88c8f004f749299b92327f6b80b", - "placeholder": "​", - "style": "IPY_MODEL_7c66f219b4aa4d6dbc88c3c6482b8f6a", - "value": " 350/350 [00:17<00:00, 20.48it/s, loss=0.17, acc=0.95, rec=0.00789, prec=0.275, f1=0.051]" - } - }, - "5056faf1de6c4adb9ed502663996d03c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "50b9017771f44898a616eee154fb927f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "56f36af5fbf54086b712b7ef0d24bfd1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5794bcad8d1d4db08d44aeb3974d4aa3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "57d9a09fbf34412d94bd3e33e8047721": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_eb49145ef1b84cd090a567775e2dcca4", - "placeholder": "​", - "style": "IPY_MODEL_5e1ef7becded4935aedcc512ae2bdf06", - "value": " 350/350 [00:04<00:00, 72.60it/s, loss=0.15, acc=0.951, rec=0.11, prec=0.661, f1=0.195]" - } - }, - "59be17e552ee4c52ad8d090cd757a1fa": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "59d9c9a838fe4f469fd7cf46f3d7d141": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5b1f3b14ddb24027bdfbeec9f2768544": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5cb237cfc9714d85ae6454b540111757": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5ccbee6e6cfc42ddbd933d967d282b46": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e8fb9b1f5bc5415a97c373bb8246201d", - "placeholder": "​", - "style": "IPY_MODEL_0e04aab9e5f64f9a92a2cb94725e11c2", - "value": " 41/41 [00:12<00:00, 3.32it/s, loss=0.205, acc=0.945, rec=0.197, prec=0.418, f1=0.264]" - } - }, - "5db09a6d3b334b529efbb282cde1c032": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "5e1ef7becded4935aedcc512ae2bdf06": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "60ea3fe7caf04d31accd1bc8db54ac16": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "636c48598d964ebf8936f9c4c3116195": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_3_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_60ea3fe7caf04d31accd1bc8db54ac16", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_5db09a6d3b334b529efbb282cde1c032", - "value": 41 - } - }, - "640cb7ed83134c11be42092a5b80a9bd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9db32a571b9a4a7fa5e82bdd4117017c", - "IPY_MODEL_3a6ac8bbac6c42928b140c731b7c3131" - ], - "layout": "IPY_MODEL_6ddb47b1c0d54509ae161ae0e5eb57ef" - } - }, - "64fc920a572140aeade0a0dc84a0d3e2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6502e4fc8fc9410082a2d3d84cca722e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "66dca67e204d4eb7a470d3a9bfd8b120": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_6_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_c69df9a5516546fd9584fa6db2fe24ba", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_97a36ec249cd420bacf166c8a4fcd1b9", - "value": 350 - } - }, - "697b57a2c79a4ad9981f7a921defd083": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_5_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_1c01671c50d2417483279004a9a979b3", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3bfebf4c1fd24fb3addd88a828219197", - "value": 41 - } - }, - "69c6bb136f714ccfa14898964dd4971c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6c1089a49bd84992a755e2f0cd202e83": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6ddb47b1c0d54509ae161ae0e5eb57ef": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6e9871894c1849e3acc14695166ac340": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_8_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_bf18478cf93c4e9694dc5e26fddda301", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a7f47fe1ac934ee0b14a1da7c4982074", - "value": 350 - } - }, - "6faf56957e5c4d9586d0d688b9dc12f0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_2_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_9ade712373dd47959f38799478cac399", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e201f9a008264687879c6fdbfcefb4f3", - "value": 350 - } - }, - "6fc1cc63618d49eda283005ca2cda68a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "736522255904410c970dcae442b39fad": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7657a19a73b2402b8c6008d3d5f26b1b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "77838c7a9c454d2a9536c23266a8da56": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "77a13458933246e28d41bee0f34c7bcb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7afa15a7d81e4d7394d6768c7b7f59d1": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4a801477f8ef4a87be58eb69291f6f75", - "placeholder": "​", - "style": "IPY_MODEL_77838c7a9c454d2a9536c23266a8da56", - "value": " 41/41 [00:00<00:00, 65.59it/s, loss=0.233, acc=0.942, rec=0.191, prec=0.343, f1=0.249]" - } - }, - "7b77188b8a7a4221b16292ac9fc9f127": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "7bd50cb649c9411ba709b337f91e3f9b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c66f219b4aa4d6dbc88c3c6482b8f6a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7c70ffcbdfdd450985294d072aef570d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7e6955d3818148989627171f36351a74": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_0_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_865d6959c2a54192bb6c758de3c277af", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_cafe62daae274a06917731ed6d4c4e4e", - "value": 350 - } - }, - "7ebea6eafed945bb8b82a3fe2157f753": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8022c7f49b2446369e05328c79341904": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "82443487d9de468f9718fb0e4ae8f717": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_9_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_77a13458933246e28d41bee0f34c7bcb", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_90739d2d08aa40949bcc66f92ce981eb", - "value": 350 - } - }, - "82eb8ce99014431ba74309a168d0b0c9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e035c062da9443bfb229071f44db6dce", - "placeholder": "​", - "style": "IPY_MODEL_50b9017771f44898a616eee154fb927f", - "value": " 350/350 [00:11<00:00, 31.18it/s, loss=0.164, acc=0.948, rec=0.0659, prec=0.582, f1=0.129]" - } - }, - "83084bf428ac435db310fbb5c20c9568": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8349300bfc7a4b7ebe4f4b4d1cce5972": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_95b0dbec1964440088c861c13d026718", - "IPY_MODEL_1681323fbd124c5b9f0b69dc86a3c01d" - ], - "layout": "IPY_MODEL_98f5956fdc274d87af51f5865091a429" - } - }, - "8501776035de4302bfe402bad761204e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "865d6959c2a54192bb6c758de3c277af": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "86f9885066e54c80be3e8f96c01af274": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "87e3b55c749a4b1d9f438b8a8024d085": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8818291bf0bb46d28532173511181808": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d1764bf182a14644a4acc155fe4f69e7", - "placeholder": "​", - "style": "IPY_MODEL_7ebea6eafed945bb8b82a3fe2157f753", - "value": " 41/41 [00:12<00:00, 3.40it/s, loss=0.166, acc=0.95, rec=0.0141, prec=0.455, f1=0.0529]" - } - }, - "89a4488b9687455a8f57a8b03567c64d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8aba096b351f4a33889cd4fa3a8a6724": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8baa7e14b9cf42a08fbc2a9f3369e0ca": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_156c73fe0e64416dbc89c10fe3ccb8b3", - "placeholder": "​", - "style": "IPY_MODEL_2dedf80d90314a76a9ce08ea848687a3", - "value": " 41/41 [00:24<00:00, 1.70it/s, loss=0.176, acc=0.946, rec=0.152, prec=0.391, f1=0.222]" - } - }, - "8d98c0c989ff491293573248bb8af7dd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_445f5fcac5154a66889a0ec782416001", - "placeholder": "​", - "style": "IPY_MODEL_6c1089a49bd84992a755e2f0cd202e83", - "value": " 41/41 [00:04<00:00, 10.21it/s, loss=0.16, acc=0.95, rec=0.0567, prec=0.473, f1=0.114]" - } - }, - "8f8b33efd16a4059910e9b945e885451": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "90739d2d08aa40949bcc66f92ce981eb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "9309f80e873f47f5a4972fc7e821b227": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_82443487d9de468f9718fb0e4ae8f717", - "IPY_MODEL_f6f77b975ec648dd8a762bcf2564c562" - ], - "layout": "IPY_MODEL_45b5e2cd4f3d4f8b8eb673307cff78d3" - } - }, - "95403b6b9e7042cbb9cc82607dab5a12": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "95b0dbec1964440088c861c13d026718": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_7_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_8022c7f49b2446369e05328c79341904", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b5ba52ba1d3f4eea9f6e58f8df2a34e6", - "value": 350 - } - }, - "97a36ec249cd420bacf166c8a4fcd1b9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "98ae22e34c8147148e35b8c51c547de5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_66dca67e204d4eb7a470d3a9bfd8b120", - "IPY_MODEL_a7b0fc1f1e0c4820997f21816c581216" - ], - "layout": "IPY_MODEL_89a4488b9687455a8f57a8b03567c64d" - } - }, - "98f5956fdc274d87af51f5865091a429": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9ade712373dd47959f38799478cac399": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9db32a571b9a4a7fa5e82bdd4117017c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_5_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_7bd50cb649c9411ba709b337f91e3f9b", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a11076e4032f4d90b4dfe9d62911427b", - "value": 350 - } - }, - "a11076e4032f4d90b4dfe9d62911427b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "a1cbce20c9b84d1f95c89a8bd6e15221": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a2180592d3e949008ad9b4f26446ecd8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_d89b429eff9f47499bf438640d50829f", - "IPY_MODEL_7afa15a7d81e4d7394d6768c7b7f59d1" - ], - "layout": "IPY_MODEL_039a7ab348934d0588ad3963dcf0f537" - } - }, - "a2741db6f4014af08ab5327ca5f489a9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a7b0fc1f1e0c4820997f21816c581216": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5794bcad8d1d4db08d44aeb3974d4aa3", - "placeholder": "​", - "style": "IPY_MODEL_7657a19a73b2402b8c6008d3d5f26b1b", - "value": " 350/350 [00:23<00:00, 14.94it/s, loss=0.0974, acc=0.963, rec=0.42, prec=0.698, f1=0.523]" - } - }, - "a7f47fe1ac934ee0b14a1da7c4982074": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "a8b2117b6c1e4a4e98c4375ab25afdac": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "aff46375cf4d4c8388893162225a5928": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b04c40ef793c48cf8c631fa22e498c61": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b407e8d9ca9f4a54a6ebf6cc9cedda0a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b577b5f2540a4aa1899ec1ce58753c8a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "b5ba52ba1d3f4eea9f6e58f8df2a34e6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "b8c58f402f02417098025ed9624d3053": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b9190069ef54401faae76f05a24a1699": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b9233f1c14f64d0f936563789c85d48d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b94fb639c6f343bea33bff21108275e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "b965434228194f3c89c8c09d0a06f935": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b9fa56c535a04bc3950feb4bfd1c5683": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6e9871894c1849e3acc14695166ac340", - "IPY_MODEL_27b787ef73a04322aa773003481e2270" - ], - "layout": "IPY_MODEL_06442feaea2140eba8041e0a30fe1e80" - } - }, - "bce71498f3f64cfabbbd53195dc8d899": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "bdfaf13535b94bb8af718c267941dc9b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bf18478cf93c4e9694dc5e26fddda301": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c105d35553a546ecb2099cb431f208aa": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c2f6a420191d4bcb9d32fcece4c70d7d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_8_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_b8c58f402f02417098025ed9624d3053", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_eb423ee397164b9da97cfe2f4140a788", - "value": 41 - } - }, - "c463c3b9fc774dbc9cc7a311eb297fe9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_4_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_64fc920a572140aeade0a0dc84a0d3e2", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6fc1cc63618d49eda283005ca2cda68a", - "value": 41 - } - }, - "c5c8dd3bba1e4ec3940c2217fc654848": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c69df9a5516546fd9584fa6db2fe24ba": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c707a9347c1747fd894259555ca3b828": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_4_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_fea2b30197654b1aad7c68b5479274fc", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b94fb639c6f343bea33bff21108275e0", - "value": 350 - } - }, - "cafe62daae274a06917731ed6d4c4e4e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "ce8e9dfbe8c24b4b8453a83073a4e27b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cea5f378b3864d8386ee467d115c3aa0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_7e6955d3818148989627171f36351a74", - "IPY_MODEL_4dc1dbf7e51b4e2a8d3c1f99a4966be4" - ], - "layout": "IPY_MODEL_3d3a021474bc4d8ea62899e335576b4d" - } - }, - "cf2d921ed3ff403c8961ce4e1ef51af0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d0e4d29a658a47a1a8ec581f41deb332": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_95403b6b9e7042cbb9cc82607dab5a12", - "placeholder": "​", - "style": "IPY_MODEL_7c70ffcbdfdd450985294d072aef570d", - "value": " 41/41 [00:18<00:00, 2.22it/s, loss=0.186, acc=0.947, rec=0.186, prec=0.431, f1=0.256]" - } - }, - "d1764bf182a14644a4acc155fe4f69e7": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d89b429eff9f47499bf438640d50829f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_9_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_a1cbce20c9b84d1f95c89a8bd6e15221", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b577b5f2540a4aa1899ec1ce58753c8a", - "value": 41 - } - }, - "da15936127e144a29ca03dde03a2a8f5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_2_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_c105d35553a546ecb2099cb431f208aa", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_20c48c3450914d5d971a179de0a14d9d", - "value": 41 - } - }, - "da248207c8d8415abb0506ff97b949e8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_59be17e552ee4c52ad8d090cd757a1fa", - "placeholder": "​", - "style": "IPY_MODEL_b407e8d9ca9f4a54a6ebf6cc9cedda0a", - "value": " 41/41 [00:35<00:00, 1.16it/s, loss=0.16, acc=0.95, rec=0.105, prec=0.46, f1=0.173]" - } - }, - "dc0aa9642d0f4e709b7062b7b1a3f9ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5b1f3b14ddb24027bdfbeec9f2768544", - "placeholder": "​", - "style": "IPY_MODEL_cf2d921ed3ff403c8961ce4e1ef51af0", - "value": " 350/350 [00:40<00:00, 8.63it/s, loss=0.138, acc=0.953, rec=0.196, prec=0.632, f1=0.295]" - } - }, - "dc5cbd8b675d4408b48a8de5f87bfcdb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_697b57a2c79a4ad9981f7a921defd083", - "IPY_MODEL_8baa7e14b9cf42a08fbc2a9f3369e0ca" - ], - "layout": "IPY_MODEL_ce8e9dfbe8c24b4b8453a83073a4e27b" - } - }, - "ddf75d040a4d477194493409f8fd26c4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f55db32f3b704caba60a19f4c00e42f8", - "IPY_MODEL_dc0aa9642d0f4e709b7062b7b1a3f9ad" - ], - "layout": "IPY_MODEL_69c6bb136f714ccfa14898964dd4971c" - } - }, - "de17fc424fdb47fe844224df94c5b939": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_0_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_fa1802cd88634c72a0cf22782d630e57", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6502e4fc8fc9410082a2d3d84cca722e", - "value": 41 - } - }, - "e035c062da9443bfb229071f44db6dce": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e0aa166b52744b0eb1f82ab92f203d2f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8aba096b351f4a33889cd4fa3a8a6724", - "placeholder": "​", - "style": "IPY_MODEL_aff46375cf4d4c8388893162225a5928", - "value": " 41/41 [00:02<00:00, 15.59it/s, loss=0.219, acc=0.942, rec=0.186, prec=0.345, f1=0.244]" - } - }, - "e1bdc8b646984fb188b48c1246a37290": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c2f6a420191d4bcb9d32fcece4c70d7d", - "IPY_MODEL_e0aa166b52744b0eb1f82ab92f203d2f" - ], - "layout": "IPY_MODEL_0612263531e14be192de9ee979e0f462" - } - }, - "e201f9a008264687879c6fdbfcefb4f3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "e46d5040c1fb456e873d9725184a03b4": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e5e87ffb3f4148f0a58d9d6b29c47156": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e8f95ab5b4fe4268abf6ca23405ba3c7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_56f36af5fbf54086b712b7ef0d24bfd1", - "placeholder": "​", - "style": "IPY_MODEL_87e3b55c749a4b1d9f438b8a8024d085", - "value": " 350/350 [00:34<00:00, 10.05it/s, loss=0.132, acc=0.954, rec=0.243, prec=0.668, f1=0.351]" - } - }, - "e8fb9b1f5bc5415a97c373bb8246201d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "eb423ee397164b9da97cfe2f4140a788": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "eb49145ef1b84cd090a567775e2dcca4": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f20173f038c840f78d9329b78872b472": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f55db32f3b704caba60a19f4c00e42f8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_3_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_736522255904410c970dcae442b39fad", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_7b77188b8a7a4221b16292ac9fc9f127", - "value": 350 - } - }, - "f60d1fa3dcbe4b419f39a44689bf3904": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "f6f77b975ec648dd8a762bcf2564c562": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_59d9c9a838fe4f469fd7cf46f3d7d141", - "placeholder": "​", - "style": "IPY_MODEL_a2741db6f4014af08ab5327ca5f489a9", - "value": " 350/350 [00:04<00:00, 70.62it/s, loss=0.0655, acc=0.975, rec=0.656, prec=0.807, f1=0.721]" - } - }, - "f976fd0613eb4a3e8efc972fcec07f16": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_48ea76e093c346f2a65c7f667d7635eb", - "IPY_MODEL_5ccbee6e6cfc42ddbd933d967d282b46" - ], - "layout": "IPY_MODEL_e5e87ffb3f4148f0a58d9d6b29c47156" - } - }, - "fa1802cd88634c72a0cf22782d630e57": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fe8b438ac1504c84baa784656b934d48": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "⭐[ep_1_i_339]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_e46d5040c1fb456e873d9725184a03b4", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_bce71498f3f64cfabbbd53195dc8d899", - "value": 350 - } - }, - "fea2b30197654b1aad7c68b5479274fc": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ff6dfefcf7aa4033a8df49f42a1ecab3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "😎[val_ep_1_i_40]: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_b9233f1c14f64d0f936563789c85d48d", - "max": 41, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f60d1fa3dcbe4b419f39a44689bf3904", - "value": 41 - } - } - }, - "version_major": 2, - "version_minor": 0 - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/dataframe_pipeline.ipynb b/nbs/dataframe_pipeline.ipynb deleted file mode 100644 index e4a5b86..0000000 --- a/nbs/dataframe_pipeline.ipynb +++ /dev/null @@ -1,1443 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DataFrame PipeLine\n", - "> A pandas dataframe pipeline\n", - "\n", - "This is a tutorial about tabular data processing pipeline" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp pipe" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dependence\n", - "This tool works on pandas DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import numpy as np\n", - "import pandas as pd\n", - "from itertools import chain\n", - "from collections import Counter" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Structure" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* ```Node``` is the basic root structure of pipeline, it's a status of data\n", - "* ```Edge``` is the **change** between the nodes, usually a function" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Node\n", - "\n", - "* Node, works on a DataFrame\n", - "* chunkNode, when the entire DataFrame is intimidating to the memory, a chunkNode works on a generator" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class Node(object):\n", - " def __init__(self, df, verbose=1):\n", - " super(Node,self).__init__()\n", - " self.df = df\n", - " self.verbose = verbose\n", - " self.pipenames = list()\n", - " self.pipe = list()\n", - "\n", - " def __repr__(self):\n", - " return \"\\n\\t|\"+\"\\n\\t|\".join(self.pipenames)\n", - "\n", - " def __or__(self, edge):\n", - " \"\"\"\n", - " use it as:\n", - " node|edge|edge\n", - " :param process_step:\n", - " :return:\n", - " \"\"\"\n", - " self.pipe.append(edge)\n", - " self.pipenames.append(edge.edge_name)\n", - " return self\n", - "\n", - " def run(self):\n", - " for pro_i in range(len(self.pipe)):\n", - " if self.verbose > 0: print(\"[df edge]:%s\" % self.pipenames[pro_i])\n", - " pro = self.pipe[pro_i]\n", - " self.df = pro.pro(self.df)\n", - " return self.df\n", - "\n", - "class chunkNode(Node):\n", - " def __init__(self, df_chunk, verbose=1):\n", - " \"\"\"\n", - " Use this class instead of Node class, for huge data sourse like big csv or huge SQL table\n", - " chunkNode(pd.read_csv(\"xxx.csv\",chunksize = 1000), verbose =1)\n", - " :param df_chunk: pandas dataframe with chunksize parameter\n", - " :param verbose:\n", - " \"\"\"\n", - " super(chunkNode, self).__init__(df = df_chunk, verbose = verbose)\n", - " self.df_chunk = self.df\n", - "\n", - " def run(self):\n", - " \"\"\"\n", - " Running iterations on the entire dataset\n", - " :return: None\n", - " \"\"\"\n", - " for df in self.df:\n", - " for pro_i in range(len(self.pipe)):\n", - " if self.verbose > 0: print(\"[df edge]:%s\" % self.pipenames[pro_i])\n", - " pro = self.pipe[pro_i]\n", - " df = pro.pro(df)\n", - "\n", - " def testrun(self):\n", - " \"\"\"\n", - " testing for 1 iteration\n", - " :return: the result dataframe\n", - " \"\"\"\n", - " testdf = next(self.df)\n", - " print(\"Please restart the generator after running test\",flush=True)\n", - " for pro_i in range(len(self.pipe)):\n", - " if self.verbose > 0: print(\"[df edge]:%s\" % self.pipenames[pro_i])\n", - " pro = self.pipe[pro_i]\n", - " testdf = pro.pro(testdf)\n", - "\n", - " return testdf" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Edges\n", - "There are 2 types of edges" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class frameEdge(object):\n", - " def __init__(self, edge_name=None):\n", - " super(frameEdge, self).__init__()\n", - " if edge_name == None:\n", - " edge_name = \"DataFrame_Processing_Edge\"\n", - " self.edge_name = edge_name\n", - " self.i = None\n", - "\n", - " def __mul__(self, cols):\n", - " assert 0, \"Only colEdge support * columns operation\"\n", - "\n", - " def define(self, f):\n", - " def wraper(df):\n", - " return f(df)\n", - "\n", - " self.pro = wraper\n", - " return wraper\n", - "\n", - "\n", - "class colEdge(object):\n", - " def __init__(self, edge_name=None):\n", - " super().__init__()\n", - " if edge_name == None:\n", - " edge_name = \"DataSeries_Processing_Edge\"\n", - " self.edge_name = edge_name\n", - " self.cols = []\n", - "\n", - " def __mul__(self, cols):\n", - " self.cols = cols\n", - " return self\n", - "\n", - " def __mod__(self,col):\n", - " self.cols = [col]\n", - " return self\n", - "\n", - " def define(self, f):\n", - " def wraper(col):\n", - " col = f(col)\n", - " return col\n", - "\n", - " self.colpro = wraper\n", - " return wraper\n", - "\n", - " def pro(self, df):\n", - " for c in self.cols:\n", - " df[c] = self.colpro(df[c])\n", - " return df\n", - "\n", - "nothing = frameEdge(\"empety_process\")\n", - "\n", - "@nothing.define\n", - "def donothing(df):\n", - " return df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* A frameEdge defines what happens from a dataframe to dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "grpby = frameEdge(\"groupby\")\n", - "\n", - "@grpby.define\n", - "def grpby_func(df):\n", - " return df.groupby(\"item_id\").count()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* A colEdge defines what happens from a column (data series) to a edited version of this column" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "double_value = colEdge(\"value double\")\n", - "\n", - "@double_value.define\n", - "def double_value_func(col):\n", - " return col*2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Some examples on edge" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class fillNaEdge(colEdge):\n", - " def __init__(self, fill=0.):\n", - " super().__init__(\"fillna_%s\"%(fill))\n", - " self.fill = fill\n", - "\n", - " def colpro(self, col):\n", - " return col.fillna(self.fill)\n", - "\n", - "\n", - "class engTokEdge(colEdge):\n", - " def __init__(self, tokenizer=None, max_len=None):\n", - " \"\"\"\n", - " Default using tweet tokenizer\n", - " \"\"\"\n", - " super().__init__(\"En Tokenization\")\n", - " if tokenizer ==None:\n", - " from nltk.tokenize import TweetTokenizer\n", - " self.tokenizer = TweetTokenizer()\n", - " else:\n", - " self.tokenizer = tokenizer\n", - " self.max_len = max_len\n", - "\n", - " def colpro(self, c):\n", - " return c.apply(lambda x: self.tokenizer.tokenize(x)[:self.max_len])\n", - "\n", - "eng_twt_tk = engTokEdge()\n", - "\n", - "class CNTok(colEdge):\n", - " def __init__(self):\n", - " \"\"\"\n", - " cntok = CNTok()\n", - " datanode = start_node|cntok*[\"col1\",\"col2\"]\n", - " datanode.run()\n", - " \"\"\"\n", - " super().__init__(\"chinese_tokenize\")\n", - " from jieba import cut\n", - " self.cut = cut\n", - "\n", - " def colpro(self, col):\n", - " col = col.apply(lambda x:list(self.cut(str(x))))\n", - " return col\n", - "\n", - "class capMinMaxEdge(colEdge):\n", - " def __init__(self, min_ = None, max_ = None):\n", - " super().__init__(\"cap min:%s max:%s\"%(min_,max_))\n", - " self.min_ = min_\n", - " self.max_ = max_\n", - "\n", - " def colpro(self,col):\n", - " col = np.clip(col.values,a_min = self.min_, a_max = self.max_)\n", - " return col\n", - "\n", - "class trackVocabEdge(colEdge):\n", - " \"\"\"\n", - " a colEdge\n", - " input column should contain python list\n", - " This edge will keep track a vocabulary pandas DataFrame\n", - " tck_vcb = TrackVocab()\n", - " tck_vcb.vocab is the accumulated vocabulary\n", - " \"\"\"\n", - " def __init__(self):\n", - " super().__init__(\"track vocab\")\n", - " self.vocab = pd.DataFrame({\"token\": [], \"cnt\": []})\n", - "\n", - " def colpro(self, col):\n", - " lists = list(col)\n", - " biglist = list(chain.from_iterable(lists))\n", - " self.vocab = self.combine_vocab(self.build_vocab(biglist))\n", - " return col\n", - "\n", - " def get_token_count_dict(self, full_tok):\n", - " \"\"\"count the token to a list\"\"\"\n", - " return Counter(full_tok)\n", - "\n", - " def build_vocab(self, full_tok):\n", - " ct_dict = self.get_token_count_dict(full_tok)\n", - " tk, ct = list(ct_dict.keys()), list(ct_dict.values())\n", - "\n", - " return pd.DataFrame({\"token\": tk, \"cnt\": ct})\n", - "\n", - " def combine_vocab(self, new_vocab):\n", - " combinedf = pd.concat([self.vocab, new_vocab]).groupby(\"token\").sum().reset_index()\n", - " return combinedf.sort_values(by=\"cnt\", ascending=False).reset_index().rename(columns = {\"index\":\"idx\"})\n", - "\n", - " def save_vocab(self, json_url):\n", - " self.vocab.to_json(json_url)\n", - "\n", - "\n", - "class saveCSV(frameEdge):\n", - " \"\"\"\n", - " DataFrame Edge\n", - " SaveCsv(\"/path/to/file.csv\")\n", - " \"\"\"\n", - " def __init__(self, csvpath, tocsv_conf={\"sep\": \"\\t\", \"index\":False}):\n", - " super().__init__(\"save to csv\")\n", - " self.csvpath = csvpath\n", - " self.tocsv_conf = tocsv_conf\n", - " self.header = True\n", - "\n", - " def pro(self, df):\n", - " df.to_csv(self.csvpath,header=self.header, mode=\"a\", **self.tocsv_conf)\n", - " self.header = False\n", - " return df\n", - "\n", - "class saveSQL(frameEdge):\n", - " \"\"\"\n", - " DataFrame Edge\n", - " SaveSQL(\"table_name\", con)\n", - " \"\"\"\n", - " def __init__(self, table_name, con, tosql_conf={\"index\": False, \"if_exists\":\"append\"}):\n", - " super().__init__(\"save to sql_table\")\n", - " self.table_name = table_name\n", - " self.con = con\n", - " self.tosql_conf = tosql_conf\n", - "\n", - " def pro(self, df):\n", - " df.to_sql(self.table_name, con=self.con, **self.tosql_conf)\n", - " return df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pipe in simple look\n", - "\n", - "#### Column edges\n", - "```edge``` defines a method of processing data.\n", - "* colEdge and its sub-class process column\n", - "* frameEdge and its cub_class process a dataframe (multiple columns)\n", - "\n", - "### Nodes/Edges relationship mapping\n", - "* node1 ==edge1==> node2 : \n", - " * \n", - " ```python\n", - " node2 = node1 | edge1\n", - " ```\n", - "* node1 ==edge1==>edge2==> node2 : \n", - " * \n", - " ```python\n", - " node2 = node1 | edge1 | edge2\n", - " ```\n", - " * \n", - " ```python\n", - " node3 = node2 | edge3\n", - " ```\n", - " \n", - "### Relationship mapping with **columns specified**\n", - "* Specifying 1 column" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "node2 = node1 | edge1%\"column_1\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* Specifying multiple columns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "node2 = node1 | edge1*[\"column_2\",\"column_3\",\"column_4\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "from pathlib import Path\n", - "import os" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Sample dataset: [New York City Airbnb Open Data](https://www.kaggle.com/dgomonov/new-york-city-airbnb-open-data)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "HOME = Path(os.environ[\"HOME\"])/\"data\"\n", - "DATA = HOME/\"AB_NYC_2019.csv\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A preview on data set" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamehost_idhost_nameneighbourhood_groupneighbourhoodlatitudelongituderoom_typepriceminimum_nightsnumber_of_reviewslast_reviewreviews_per_monthcalculated_host_listings_countavailability_365
02539Clean & quiet apt home by the park2787JohnBrooklynKensington40.64749-73.97237Private room149192018-10-190.216365
12595Skylit Midtown Castle2845JenniferManhattanMidtown40.75362-73.98377Entire home/apt2251452019-05-210.382355
23647THE VILLAGE OF HARLEM....NEW YORK !4632ElisabethManhattanHarlem40.80902-73.94190Private room15030NaNNaN1365
33831Cozy Entire Floor of Brownstone4869LisaRoxanneBrooklynClinton Hill40.68514-73.95976Entire home/apt8912702019-07-054.641194
45022Entire Apt: Spacious Studio/Loft by central park7192LauraManhattanEast Harlem40.79851-73.94399Entire home/apt801092018-11-190.1010
\n", - "
" - ], - "text/plain": [ - " id name host_id \\\n", - "0 2539 Clean & quiet apt home by the park 2787 \n", - "1 2595 Skylit Midtown Castle 2845 \n", - "2 3647 THE VILLAGE OF HARLEM....NEW YORK ! 4632 \n", - "3 3831 Cozy Entire Floor of Brownstone 4869 \n", - "4 5022 Entire Apt: Spacious Studio/Loft by central park 7192 \n", - "\n", - " host_name neighbourhood_group neighbourhood latitude longitude \\\n", - "0 John Brooklyn Kensington 40.64749 -73.97237 \n", - "1 Jennifer Manhattan Midtown 40.75362 -73.98377 \n", - "2 Elisabeth Manhattan Harlem 40.80902 -73.94190 \n", - "3 LisaRoxanne Brooklyn Clinton Hill 40.68514 -73.95976 \n", - "4 Laura Manhattan East Harlem 40.79851 -73.94399 \n", - "\n", - " room_type price minimum_nights number_of_reviews last_review \\\n", - "0 Private room 149 1 9 2018-10-19 \n", - "1 Entire home/apt 225 1 45 2019-05-21 \n", - "2 Private room 150 3 0 NaN \n", - "3 Entire home/apt 89 1 270 2019-07-05 \n", - "4 Entire home/apt 80 10 9 2018-11-19 \n", - "\n", - " reviews_per_month calculated_host_listings_count availability_365 \n", - "0 0.21 6 365 \n", - "1 0.38 2 355 \n", - "2 NaN 1 365 \n", - "3 4.64 1 194 \n", - "4 0.10 1 0 " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.read_csv(DATA)\n", - "\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Start node, a pandas dataframe" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Real Data Example" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.pipe import *" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "start_node = Node(df)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "cap_minmax_edge = capMinMaxEdge(max_ = 100)\n", - "fill_na_edge = fillNaEdge(\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Setting the node/edge pipeline" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "# clip minmax value on 1 column\n", - "# fill na to \"\", on 2 columns\n", - "# tokenize, on 2 columns\n", - "end_node = start_node|cap_minmax_edge %\"number_of_reviews\"\\\n", - " |fill_na_edge *[\"name\",\"room_type\"]\\\n", - " |eng_twt_tk*[\"name\",\"room_type\"] " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Print out the pipeline layout" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "\t|cap min:None max:100\n", - "\t|fillna_\n", - "\t|En Tokenization" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "end_node" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Excute the processing" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n" - ] - } - ], - "source": [ - "end_df = end_node.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Checking the processed data" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamehost_idhost_nameneighbourhood_groupneighbourhoodlatitudelongituderoom_typepriceminimum_nightsnumber_of_reviewslast_reviewreviews_per_monthcalculated_host_listings_countavailability_365
02539[Clean, &, quiet, apt, home, by, the, park]2787JohnBrooklynKensington40.64749-73.97237[Private, room]149192018-10-190.216365
12595[Skylit, Midtown, Castle]2845JenniferManhattanMidtown40.75362-73.98377[Entire, home, /, apt]2251452019-05-210.382355
23647[THE, VILLAGE, OF, HARLEM, ..., NEW, YORK, !]4632ElisabethManhattanHarlem40.80902-73.94190[Private, room]15030NaNNaN1365
33831[Cozy, Entire, Floor, of, Brownstone]4869LisaRoxanneBrooklynClinton Hill40.68514-73.95976[Entire, home, /, apt]8911002019-07-054.641194
45022[Entire, Apt, :, Spacious, Studio, /, Loft, by...7192LauraManhattanEast Harlem40.79851-73.94399[Entire, home, /, apt]801092018-11-190.1010
\n", - "
" - ], - "text/plain": [ - " id name host_id \\\n", - "0 2539 [Clean, &, quiet, apt, home, by, the, park] 2787 \n", - "1 2595 [Skylit, Midtown, Castle] 2845 \n", - "2 3647 [THE, VILLAGE, OF, HARLEM, ..., NEW, YORK, !] 4632 \n", - "3 3831 [Cozy, Entire, Floor, of, Brownstone] 4869 \n", - "4 5022 [Entire, Apt, :, Spacious, Studio, /, Loft, by... 7192 \n", - "\n", - " host_name neighbourhood_group neighbourhood latitude longitude \\\n", - "0 John Brooklyn Kensington 40.64749 -73.97237 \n", - "1 Jennifer Manhattan Midtown 40.75362 -73.98377 \n", - "2 Elisabeth Manhattan Harlem 40.80902 -73.94190 \n", - "3 LisaRoxanne Brooklyn Clinton Hill 40.68514 -73.95976 \n", - "4 Laura Manhattan East Harlem 40.79851 -73.94399 \n", - "\n", - " room_type price minimum_nights number_of_reviews \\\n", - "0 [Private, room] 149 1 9 \n", - "1 [Entire, home, /, apt] 225 1 45 \n", - "2 [Private, room] 150 3 0 \n", - "3 [Entire, home, /, apt] 89 1 100 \n", - "4 [Entire, home, /, apt] 80 10 9 \n", - "\n", - " last_review reviews_per_month calculated_host_listings_count \\\n", - "0 2018-10-19 0.21 6 \n", - "1 2019-05-21 0.38 2 \n", - "2 NaN NaN 1 \n", - "3 2019-07-05 4.64 1 \n", - "4 2018-11-19 0.10 1 \n", - "\n", - " availability_365 \n", - "0 365 \n", - "1 355 \n", - "2 365 \n", - "3 194 \n", - "4 0 " - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "end_df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Working with bigger data\n", - "There is often the cases when we have to deal with csv > 500m in size or a huge sql table.\n", - "\n", - "In some cases we want to work/ save the data batch by batch\n", - "\n", - "Gladely, pandas offer a chunksize solution, so we can process huge structured data in batch(defined by the chunk size)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.pipe import chunkNode,saveCSV, saveSQL" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```saveCSV``` : saving file could be a part of the pipeline, we can follow that edge by ```saveSQL``` if we like" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "start_node = chunkNode(pd.read_csv(DATA, chunksize=5000), verbose = 1)\n", - "\n", - "end_node = start_node|cap_minmax_edge %\"number_of_reviews\"\\\n", - " |fill_na_edge *[\"name\",\"room_type\"]\\\n", - " |eng_twt_tk*[\"name\",\"room_type\"]\\\n", - " |saveCSV(HOME/\"nyc_processed.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pipeline layout summary" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "\t|cap min:None max:100\n", - "\t|fillna_\n", - "\t|En Tokenization\n", - "\t|save to csv" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "end_node" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Notice**\n", - "* ```run``` function that \n", - " * start with a chunkNode has **no return**, \n", - " * start with a Node will ruturn the **result dataframe**. \n", - "* This feature is purposefully designed, assuming the result data could also be huge and not suitable to remain its entirety in RAM." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Excute the processing， if you are annoyed by the verbosity, set ```verbose=0```" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n", - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:save to csv\n" - ] - } - ], - "source": [ - "end_node.run()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\t2539\t['Clean', '&', 'quiet', 'apt', 'home', 'by', 'the', 'park']\t2787\tJohn\tBrooklyn\tKensington\t40.647490000000005\t-73.97237\t['Private', 'room']\t149\t1\t9\t2018-10-19\t0.21\t6\t365\r\n", - "1\t2595\t['Skylit', 'Midtown', 'Castle']\t2845\tJennifer\tManhattan\tMidtown\t40.75362\t-73.98376999999999\t['Entire', 'home', '/', 'apt']\t225\t1\t45\t2019-05-21\t0.38\t2\t355\r\n", - "2\t3647\t['THE', 'VILLAGE', 'OF', 'HARLEM', '...', 'NEW', 'YORK', '!']\t4632\tElisabeth\tManhattan\tHarlem\t40.809020000000004\t-73.9419\t['Private', 'room']\t150\t3\t0\t\t\t1\t365\r\n" - ] - } - ], - "source": [ - "!head -n3 {HOME/\"nyc_processed.csv\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a new edge\n", - "Create a new edge\n", - "\n", - "Define the processing function with the ```define``` decorator.\n", - "\n", - "* col is a pandas data series, the concept of ```column``` in pandas\n", - "* In this case we use the ```apply``` function of data series, any decorated function would work as long as it return another data series" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "lower_case = colEdge(\"lower case\")\n", - "\n", - "\n", - "def lowerList(x):\n", - " return list(str(i).lower() for i in x)\n", - "\n", - "@lower_case.define\n", - "def lower(col):\n", - " return col.apply(lowerList)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# The DIYed edge will work on columns \"name\" and \"room_type\" after tokenization\n", - "df = pd.read_csv(DATA)\n", - "start_node = Node(df)\n", - "end_node = start_node|cap_minmax_edge %\"number_of_reviews\"\\\n", - " |fill_na_edge *[\"name\",\"room_type\"]\\\n", - " |eng_twt_tk*[\"name\",\"room_type\"]\\\n", - " |lower_case*[\"name\",\"room_type\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "\t|cap min:None max:100\n", - "\t|fillna_\n", - "\t|En Tokenization\n", - "\t|lower case" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "end_node" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[df edge]:cap min:None max:100\n", - "[df edge]:fillna_\n", - "[df edge]:En Tokenization\n", - "[df edge]:lower case\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idnamehost_idhost_nameneighbourhood_groupneighbourhoodlatitudelongituderoom_typepriceminimum_nightsnumber_of_reviewslast_reviewreviews_per_monthcalculated_host_listings_countavailability_365
02539[clean, &, quiet, apt, home, by, the, park]2787JohnBrooklynKensington40.64749-73.97237[private, room]149192018-10-190.216365
12595[skylit, midtown, castle]2845JenniferManhattanMidtown40.75362-73.98377[entire, home, /, apt]2251452019-05-210.382355
23647[the, village, of, harlem, ..., new, york, !]4632ElisabethManhattanHarlem40.80902-73.94190[private, room]15030NaNNaN1365
33831[cozy, entire, floor, of, brownstone]4869LisaRoxanneBrooklynClinton Hill40.68514-73.95976[entire, home, /, apt]8911002019-07-054.641194
45022[entire, apt, :, spacious, studio, /, loft, by...7192LauraManhattanEast Harlem40.79851-73.94399[entire, home, /, apt]801092018-11-190.1010
\n", - "
" - ], - "text/plain": [ - " id name host_id \\\n", - "0 2539 [clean, &, quiet, apt, home, by, the, park] 2787 \n", - "1 2595 [skylit, midtown, castle] 2845 \n", - "2 3647 [the, village, of, harlem, ..., new, york, !] 4632 \n", - "3 3831 [cozy, entire, floor, of, brownstone] 4869 \n", - "4 5022 [entire, apt, :, spacious, studio, /, loft, by... 7192 \n", - "\n", - " host_name neighbourhood_group neighbourhood latitude longitude \\\n", - "0 John Brooklyn Kensington 40.64749 -73.97237 \n", - "1 Jennifer Manhattan Midtown 40.75362 -73.98377 \n", - "2 Elisabeth Manhattan Harlem 40.80902 -73.94190 \n", - "3 LisaRoxanne Brooklyn Clinton Hill 40.68514 -73.95976 \n", - "4 Laura Manhattan East Harlem 40.79851 -73.94399 \n", - "\n", - " room_type price minimum_nights number_of_reviews \\\n", - "0 [private, room] 149 1 9 \n", - "1 [entire, home, /, apt] 225 1 45 \n", - "2 [private, room] 150 3 0 \n", - "3 [entire, home, /, apt] 89 1 100 \n", - "4 [entire, home, /, apt] 80 10 9 \n", - "\n", - " last_review reviews_per_month calculated_host_listings_count \\\n", - "0 2018-10-19 0.21 6 \n", - "1 2019-05-21 0.38 2 \n", - "2 NaN NaN 1 \n", - "3 2019-07-05 4.64 1 \n", - "4 2018-11-19 0.10 1 \n", - "\n", - " availability_365 \n", - "0 365 \n", - "1 355 \n", - "2 365 \n", - "3 194 \n", - "4 0 " - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "end_df = end_node.run()\n", - "end_df.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/index.ipynb b/nbs/index.ipynb deleted file mode 100644 index 34e109b..0000000 --- a/nbs/index.ipynb +++ /dev/null @@ -1,625 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ForgeBox\n", - "> Data science comprehensive toolbox 🛠⚔️📦" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![forgebox logo](https://raw.githubusercontent.com/raynardj/forgebox/new_feature/docs/logo.jpg)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Installation\n", - "\n", - "Easy simple installation in 1 line\n", - "```shell\n", - "pip install forgebox\n", - "```\n", - "\n", - "If not specified, you need anaconda3 for most of the tools, python shold be at least >=3.6" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Features 🚀 Briefing\n", - "\n", - "> This is a tool box with comprehensive **utilies**, to put it simply, I just hope most of my frequetyly used DIY tools in in place and can be easily installed and imported" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Lazy, fast imports 🤯" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following command will import many frequent tools for data science, like **pd** for pandas, **np** for numpy, os, json, PIL.Image for image processing\n", - "\n", - "```python\n", - "from frogebox.imports import *\n", - "```\n", - "\n", - "No more🚫 following typings\n", - "```python\n", - "import pandas as pd\n", - "import numpy as np\n", - "import os\n", - "import json\n", - "...\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Categorical converter\n", - "\n", - "> Mapping and converting categorical infomation" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.category import Category" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n" - ] - } - ], - "source": [ - "az = list(map(chr,range(ord(\"A\"), ord(\"z\")+1)))\n", - "print(az)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Category Manager with 58" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cate_az = Category(az)\n", - "cate_az" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(17, 'R')" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cate_az.c2i[\"R\"], cate_az.i2c[17]" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 5, 46, 49, 38, 36, 1, 46, 55])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cate_az.c2i[list(\"ForgeBox\")]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['F', 'o', 'r', 'g', 'e', 'B', 'o', 'x'], dtype='\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathfile_typeparentdepth
36/Users/xiaochen.zhang/.cache/torch/transformer...jsontransformers7
13/Users/xiaochen.zhang/.cache/torch/transformer...jsontransformers7
51/Users/xiaochen.zhang/.cache/langhuan/task_NER...jsontask_NER_210121_1405137
32/Users/xiaochen.zhang/.cache/torch/transformer...locktransformers7
58/Users/xiaochen.zhang/.cache/langhuan/task_Cla...jsontask_Classify_210128_1647107
\n", - "" - ], - "text/plain": [ - " path file_type parent depth\n", - "36 /Users/xiaochen.zhang/.cache/torch/transformer... json transformers 7\n", - "13 /Users/xiaochen.zhang/.cache/torch/transformer... json transformers 7\n", - "51 /Users/xiaochen.zhang/.cache/langhuan/task_NER... json task_NER_210121_140513 7\n", - "32 /Users/xiaochen.zhang/.cache/torch/transformer... lock transformers 7\n", - "58 /Users/xiaochen.zhang/.cache/langhuan/task_Cla... json task_Classify_210128_164710 7" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "file_detail(\"/Users/xiaochen.zhang/.cache/\").sample(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JS style async" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.asyncing import Async\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1111111111\n", - "22222222222\n", - "[result]:\tslept for 2 seconds\n" - ] - } - ], - "source": [ - "def something_time_costing_but_you_dont_want_to_wait(x):\n", - " sleep(x)\n", - " return f\"slept for {x} seconds\"\n", - "\n", - "def task2_you_will_perfrom_after_the_time_costing_one(x):\n", - " print(f\"[result]:\\t{x}\")\n", - " return 1\n", - "\n", - "print(\"1111111111\")\n", - "\n", - "Async(something_time_costing_but_you_dont_want_to_wait)(2)\\\n", - ".then(task2_you_will_perfrom_after_the_time_costing_one)\\\n", - ".catch(print)\n", - "\n", - "print(\"22222222222\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### HTML in notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.html import DOM, list_group, list_group_kv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This will map a clear HTML table view of wild tree type json structure/ list" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
  • police
  • headpin
    • acdc
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "bands = [\"police\", \"headpin\", {\"ac\":\"dc\"}]\n", - "list_group(bands)()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
  • questionanswer
  • another
    • deeper questionanswer again
  • final
    • questions
      • what
      • is
      • the
      • answer
      • to
        • life
        • universe
        • everything
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "questions = {\n", - " \"question\":\"answer\",\n", - " \"another\":{\n", - " \"deeper question\": \"answer again\"},\n", - " \"final\":{\n", - " \"questions\": [\"what\",\"is\",\"the\",\"answer\", \"to\",\n", - " [\"life\", \"universe\",\"everything\"]]}\n", - "}\n", - "list_group_kv(questions)()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Coding html in python" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Title example
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
  • Line 0
  • Line 1
  • Line 2
  • Line 3
  • Line 4
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "title = DOM(\"Title example\",\"h5\", kwargs={\"style\":\"color:#3399EE\"})\n", - "ul = DOM(\"\",\"ul\");\n", - "for i in range(5):\n", - " ul = ul.append(DOM(f\"Line {i}\", \"li\", kwargs={\"style\":\"color:#EE33DD\"}))\n", - "\n", - "title()\n", - "ul()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Free style mapping" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Works on every value of a complicated dictionary structure (eg. list in dict in list in dict, etc,. 😳)\n", - "\n", - "```python\n", - "from forgebox.freemap import FreeMap\n", - "\n", - "# flatten decides if we want to flatten the strucuture\n", - "freemap_tool = FreeMap(\n", - " ,\n", - " ,\n", - " flatten=True\n", - ")\n", - "\n", - "data2 = freemap_tool(data1)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Interactive Widgets\n", - "> Interactive widgets work with in jupyter notebooks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Search box 🔎 for dataframe\n", - "This will create an interactive text input box to search through the pandas dataframe, within the columns you set.\n", - "\n", - "if ```manual``` is set to False, the search will respond to **each of your key press**, it's fast but will suffer terrible user experience if the dataframe is huge in size.\n", - "\n", - "```python\n", - "from forgebox.widgets import search_box\n", - "\n", - "search_box(data_df, columns=[\"col1\",\"col2\"], manual=False)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### paginate\n", - "You can browse through a pandas dataframe like fliping pages 📄.\n", - "\n", - "```python\n", - "from forgebox.widgets import paginate\n", - "\n", - "paginate(your_dataframe, page_len=10)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Single button callback\n", - "> a fully functional page with a single button, this single button is bonded to a function\n", - "\n", - "This is as much code as you need, to build a fully functional interactive page shows sql table from jupyter, that you can:\n", - "\n", - "* choose which table to visit\n", - "* choose how many lines you want to show, (with a slider)\n", - "* configure the where condition with a text box on front end\n", - "\n", - "```python\n", - "tablename_list = [\"pubmed\", \"patient\", \"users\", \"drugs\"]\n", - "\n", - "from forgebox.html import DOM\n", - "def show_sql_table(sql_input:str) -> str:\n", - " with engine.connect() as conn:\n", - " df=pd.read_sql(sql_input, con=conn)\n", - " # display the table as html\n", - " DOM(df.to_html(),\"div\")()\n", - "\n", - "@SingleButton(callback=show_sql_table)\n", - "def abc(\n", - " limit:{\"typing\":int, \"default\":10, \"min\":5, \"max\":20},\n", - " where_condition:{\"typing\":str, \"default\": \"where 1=1\", },\n", - " table:{\"typing\":list, \"options\":tablename_list}\n", - "):\n", - " return f\"SELECT * FROM {table} {where_condition} LIMIT {limit}\"\n", - "```" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/test.json b/nbs/test.json deleted file mode 100644 index 5a4a870..0000000 --- a/nbs/test.json +++ /dev/null @@ -1 +0,0 @@ -{"a": 1, "b": 2, "c": 5, "d": 6, "e": 12} \ No newline at end of file diff --git a/settings.ini b/settings.ini index f603cd9..76b7aed 100644 --- a/settings.ini +++ b/settings.ini @@ -7,7 +7,7 @@ author = xiaochen(ray) zhang author_email = b2ray2c@gmail.com copyright = xiaochen(ray) zhang branch = master -version = 1.0.2 +version = 1.0.3 min_python = 3.6 host = github audience = Developers