diff --git a/README.md b/README.md index 113737a..91a8af1 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,17 @@ from xllm.datasets import GeneralDataset from xllm.experiments import Experiment # Init Config which controls the internal logic of xllm -config = Config(model_name_or_path="HuggingFaceH4/zephyr-7b-beta") +# QLoRA example +config = Config( + model_name_or_path="HuggingFaceH4/zephyr-7b-beta", + stabilize=True, + apply_lora=True, + load_in_4bit=True, + push_to_hub=True, + hub_private_repo=True, + hub_model_id="BobaZooba/SupaDupaZephyr-7B-LoRA", + save_steps=1_000, +) # Prepare the data train_data = ["Hello!"] * 100 diff --git "a/examples/notebooks/Llama2_&_Mistral_AI_efficient_fine_tuning_using_QLoRA,_bnb_int4,_gradient_checkpointing_and_X\342\200\224LLM_\360\237\246\226.ipynb" "b/examples/notebooks/Llama2_&_Mistral_AI_efficient_fine_tuning_using_QLoRA,_bnb_int4,_gradient_checkpointing_and_X\342\200\224LLM_\360\237\246\226.ipynb" index 80e52e5..c1cc3ba 100644 --- "a/examples/notebooks/Llama2_&_Mistral_AI_efficient_fine_tuning_using_QLoRA,_bnb_int4,_gradient_checkpointing_and_X\342\200\224LLM_\360\237\246\226.ipynb" +++ "b/examples/notebooks/Llama2_&_Mistral_AI_efficient_fine_tuning_using_QLoRA,_bnb_int4,_gradient_checkpointing_and_X\342\200\224LLM_\360\237\246\226.ipynb" @@ -1,13509 +1,13505 @@ { - "cells": [ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Llama2 & Mistral AI efficient fine-tuning using QLoRA, bnb int4, gradient checkpointing and X—LLM 🦖\n", + "\n", + "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", + "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", + "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", + "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", + " do\n", + "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", + " into your own project\n", + "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", + "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model" + ], + "metadata": { + "id": "RQQ_O9Wmsgxw" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eVEUs8X5rTiV" + }, + "source": [ + "# Installation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c4jm8Qr2qMuU", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "ba0c98e4-e657-4fae-872e-ff9ff044579f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting xllm\n", + " Downloading xllm-0.0.10-py3-none-any.whl (103 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.0/104.0 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (23.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from xllm) (5.9.5)\n", + "Requirement already satisfied: torch>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (2.1.0+cu118)\n", + "Collecting loguru (from xllm)\n", + " Downloading loguru-0.7.2-py3-none-any.whl (62 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting peft>=0.5.0 (from xllm)\n", + " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting wandb (from xllm)\n", + " Downloading wandb-0.16.0-py3-none-any.whl (2.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting python-dotenv (from xllm)\n", + " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from xllm) (2.31.0)\n", + "Collecting optimum>=1.12.0 (from xllm)\n", + " Downloading optimum-1.14.0-py3-none-any.whl (398 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m398.9/398.9 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting bitsandbytes>=0.41.1 (from xllm)\n", + " Downloading bitsandbytes-0.41.2.post2-py3-none-any.whl (92.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from xllm) (1.11.3)\n", + "Collecting transformers (from xllm)\n", + " Downloading transformers-4.35.1-py3-none-any.whl (7.9 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m40.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from xllm) (4.66.1)\n", + "Collecting safetensors (from xllm)\n", + " Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m33.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting coloredlogs (from optimum>=1.12.0->xllm)\n", + " Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (1.12)\n", + "Collecting huggingface-hub>=0.8.0 (from optimum>=1.12.0->xllm)\n", + " Downloading huggingface_hub-0.19.1-py3-none-any.whl (311 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.1/311.1 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting datasets (from optimum>=1.12.0->xllm)\n", + " Downloading datasets-2.14.6-py3-none-any.whl (493 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m35.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (6.0.1)\n", + "Collecting accelerate>=0.21.0 (from peft>=0.5.0->xllm)\n", + " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (4.5.0)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.1.2)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2023.6.0)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2.1.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2023.7.22)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (2023.6.3)\n", + "Collecting tokenizers<0.15,>=0.14 (from transformers->xllm)\n", + " Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m103.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (8.1.7)\n", + "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb->xllm)\n", + " Downloading GitPython-3.1.40-py3-none-any.whl (190 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.6/190.6 kB\u001b[0m \u001b[31m24.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting sentry-sdk>=1.0.0 (from wandb->xllm)\n", + " Downloading sentry_sdk-1.35.0-py2.py3-none-any.whl (248 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m248.6/248.6 kB\u001b[0m \u001b[31m34.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb->xllm)\n", + " Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n", + "Collecting setproctitle (from wandb->xllm)\n", + " Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (67.7.2)\n", + "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.4.4)\n", + "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.20.3)\n", + "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb->xllm) (1.16.0)\n", + "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb->xllm)\n", + " Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting huggingface-hub>=0.8.0 (from optimum>=1.12.0->xllm)\n", + " Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting sentencepiece!=0.1.92,>=0.1.91 (from transformers->xllm)\n", + " Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m88.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting humanfriendly>=9.1 (from coloredlogs->optimum>=1.12.0->xllm)\n", + " Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (9.0.0)\n", + "Collecting dill<0.3.8,>=0.3.0 (from datasets->optimum>=1.12.0->xllm)\n", + " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (1.5.3)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.4.1)\n", + "Collecting multiprocess (from datasets->optimum>=1.12.0->xllm)\n", + " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.8.6)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.1->xllm) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum>=1.12.0->xllm) (1.3.0)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (23.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.3.1)\n", + "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->xllm)\n", + " Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2023.3.post1)\n", + "Installing collected packages: sentencepiece, bitsandbytes, smmap, setproctitle, sentry-sdk, safetensors, python-dotenv, loguru, humanfriendly, docker-pycreds, dill, multiprocess, huggingface-hub, gitdb, coloredlogs, tokenizers, GitPython, accelerate, wandb, transformers, datasets, peft, optimum, xllm\n", + "Successfully installed GitPython-3.1.40 accelerate-0.24.1 bitsandbytes-0.41.2.post2 coloredlogs-15.0.1 datasets-2.14.6 dill-0.3.7 docker-pycreds-0.4.0 gitdb-4.0.11 huggingface-hub-0.17.3 humanfriendly-10.0 loguru-0.7.2 multiprocess-0.70.15 optimum-1.14.0 peft-0.6.2 python-dotenv-1.0.0 safetensors-0.4.0 sentencepiece-0.1.99 sentry-sdk-1.35.0 setproctitle-1.3.3 smmap-5.0.1 tokenizers-0.14.1 transformers-4.35.1 wandb-0.16.0 xllm-0.0.10\n" + ] + } + ], + "source": [ + "!pip install --upgrade xllm" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Login to HuggingFace to save model to the hub" + ], + "metadata": { + "id": "l_xO5kdESEUl" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c_BRWCP9FHnG" + }, + "outputs": [], + "source": [ + "# !huggingface-cli login" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# [Optional] Login to W&B to save training process" + ], + "metadata": { + "id": "ksKCydbdy5Dp" + } + }, + { + "cell_type": "code", + "source": [ + "# !wandb login" + ], + "metadata": { + "id": "8ZvYGsYXyuaY" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Tiine2-9rVpc" + }, + "source": [ + "# Prepare" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BuBIZNOQqZOX", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "c46f967a-debd-42c7-b835-b00be9316a86" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "X—LLM version: 0.0.10\n", + "Torch version: 2.1.0+cu118\n", + "Cuda is available: True\n" + ] + } + ], + "source": [ + "import torch\n", + "import xllm\n", + "\n", + "cuda_is_available = torch.cuda.is_available()\n", + "\n", + "print(f\"X—LLM version: {xllm.__version__}\\nTorch version: {torch.__version__}\\nCuda is available: {cuda_is_available}\")\n", + "assert cuda_is_available" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DbDhVyrrqbUe" + }, + "outputs": [], + "source": [ + "from xllm import Config\n", + "from xllm.datasets import GeneralDataset\n", + "from xllm.experiments import Experiment" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Prepare dataset" + ], + "metadata": { + "id": "cU3jAEAgSVmh" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BF8PadIirHHJ", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 433, + "referenced_widgets": [ + "91d2f708f17a44b6948f6098b446d8bf", + "f346818d99cb4fdebb0f41a55233b434", + "fba9313b8d00498b8e01584e628144dd", + "9e5c4bc716b247fab2fb94c33e42de97", + "f99d23f960a14df98fdbc361fc8190c4", + "b04788dba2c2440e863ef3544d38573a", + "80122fe6534b40f58c033bc05c529c4d", + "e6ec77fde8fb4f538154ff8a20862f78", + "def19bd9681b429dbeca0c5c631ae907", + "bcca153bcbb9498fb1bccf47b7952211", + "8461e3957e1d4fa9a2a0949e06e6d9d0", + "36f7f01c1d06465b9be2f2f05bd691bc", + "5e66f45e1af044c7a7e6de5bf98b4c67", + "8222bc4078b1402bbd12e698487423bc", + "b32a3ffdbdd6450fbb0c1ef2e5259a72", + "42942ded69914692b0fbacaac7dfc689", + "0e58a81fdcaf4d00916d7e8eb501083d", + "230a38c3d69346f3a5ecf58467cb3185", + "09b02581f7934e129d8475d387e4f08b", + "a7e442a2f4774421801bdd6be92c89c8", + "28e83b8d6b7b4e208942ce3ab45122a9", + "a86aef9a8efe4e26bdc64afc2eba6175", + "5e2aec219c6d47d89220d9e03e37464e", + "c3a2faa248584594aa3a49899573b758", + "18010298d8ef40189ecd32fd0b9b193e", + "5cbba4bd9fad4d98b7972f4ea1f1220e", + "9f1cb70e1fec42e6994b3dfd2eb7c4ad", + "629bcbc6f7014ad6ae426b4c909b394c", + "5ce6585f2c8445af9c7771c798abab73", + "7607b126c1fa45e4b0e6ed3d68abbf2f", + "519165fc2cb74ae9a7c669ce70559ce0", + "907880d0d0f141b2aa53171047a31ba8", + "21da78f5acb048faad6ada99f9a5c33c", + "314180cc67804414b0c5f988c792e090", + "42abc0e90af048e599425e06a537cd87", + "65447fa3d25449dfa9337005892bc487", + "d146b05b23f141cc9d78cbeda8998dfc", + "a513ce1047cc48baa963d1f29484101f", + "e037c802b3a44f018dfbb35c2b46e051", + "18841a16d4ed4a3b88a09dd8e4af327a", + "aebc3d9ba33f491396f7885b342850ba", + "32b4e1d862db40fa875f6b4cc93414fd", + "d8c4748e91bd45ab83a12c2fb8eb955b", + "ad722fd9b68c4e62885e216e674fba5b", + "75509da037034f8fbf4d843d7c4b6a9c", + "7b70293dbbba49adbe2c691485f077b7", + "397957cff0ac4ac8b361117b33c8b942", + "5fe9d1ea86c44f2b9080d61c2f01990c", + "2a8f690b4b6d425d882fd2d82a165d8e", + "96e74daee8c3417c8cf2e986d5c1587c", + "214bb43f5e324f7c88dcb237b98a77a1", + "d54cec2df24b458498308c21cbc3d718", + "6cd752c91e3a4f8f8ad1cb45c8de2e48", + "1ffe5bd186c0485ab13bd62845f16cad", + "e03c32cb5def4b6ca9f8c88a5ac87b2b", + "e156d8d19ea64cada768f9f56b7cf570", + "e4149c5fc6fe4714b0f946590ed31923", + "d02b76d5303740b889be3654f049e616", + "ce2a8b69e2fe4b2abb09b12797d49931", + "07cd7f354e4f494e89920a21088fa4ec", + "154a3dd8714747f58907d69d22fa8a20", + "67e9c5fe352e44358efcd34128182d31", + "9322bf4773304158b259a7c883eda7ef", + "c6b651c5bed5415885ba9a57bfd9436d", + "d4ab49580be641589b3e08ea86ce7de4", + "0cb08054ba764ada91b3392444ded3d9", + "474d734daecb48c5a7421ec847c55dfe", + "85293612d3664240a4830302899b5738", + "677d2ac048af473baedb1f78ba905949", + "e189bc5cf21e4011a6d7a2af30c80a72", + "3a3ce6335c23482b8ce3a1a04a6ca982", + "09157d52a2114067aa8adfa31b3d20f3", + "837c0cd06c8a47db950d89235d63e555", + "34090c60ee3b4ee1aad730923a4e8193", + "caf9d4c2fbdc40dea78a852ca9fd9fd5", + "acff29dce7e04631aa5712a2eba9d4d7", + "73e69b26be394e10a401bc5f743b282d", + "215364005ed54a208b77791bb5a99e07", + "3cb4cf22b0a14352a96a229498f7d8a9", + "75098626ea0f41489031a3bb4e82a223", + "9a78f50e1c2c443d87d3c64269caea5d", + "6fa26245072342f0bc035e0eb5cf84a1", + "7527883a7c3a4bfd9e11f02e04c16b0c", + "79684b9cf9e348879d6ab067b7f2c090", + "f7f5e305cce747b69c2fd3b82db3f3ca", + "6eab232a2238471e9ca13454493f642d", + "99e684c348164317b912454a8815c843", + "da73ad3833e04476aba0039617159dba", + "44e21fb7312d413d9b7918d5cfdb7585", + "c1551efd35304fc6a154f926166d36a2", + "4ad74ec101f44e31bfb17a5944222d13", + "041cd1486bdd4de5a719a952c575a6c7", + "557f05840029443983f472cf47c65036", + "d22e441c274544b2b433a3b01b82aa26", + "09c8ffc919204833a4bae35f5589bd2f", + "b8ae603869a6454a81b8b9010e8f102d", + "79fffc369ea14b4bbc48be794427eefb", + "11fc484f96f04982b370010f4fbd433d", + "9f3955a94ebb4d1c9e36b92e0064255d", + "a62f49e21861447ea4e234ac3d0c32fb", + "ffeeba165a944ace83699a6e750a7585", + "bd247663f40c460ca7b36d6918154227", + "47143b289474470eb7ca3e076f713e40", + "4858305b3b64403db561eb39ef42cdf5", + "193a85fe0ec045af858cf39a2bc1f14b", + "8e982a2b62284297897faca842ec12b3", + "e3827c994b12424db605249d876a2607", + "4d25d9dfa2834616aff78a6d1036d406", + "2a44d9d3172b4b88bb1d192573e802fd", + "fd2f9eb5bd304816b1bff17f97894c3f", + "0b3c98d4bb3a4808b47e7449bba82c3f", + "df60313f9b6747f3a5f52533f78edbe2", + "596ae6b733c847b9a27c5d84a30fae0b", + "4b524b9d0516423fb1d51bc8402a38b2", + "0798c26f90b84956be0a7f5116c630b6", + "952758db560743b79eefaaf3a063ed46", + "6a26d8b4cf244b759311d4c2e219228e", + "cbd796b765a446e1af48954203291985", + "6880ca900dbe4a02902dd21d98234057", + "2fc769756af74848ac6c2b24e35707b7", + "c0563be78a9d405eafc7907a07129282", + "83cf8c6b915444b1baca520427f8309e", + "0826c34ece2b4ed9b88896083a8e5c68", + "aaf4c83c6795485493448a8148ac674f", + "394649d55fe54e2b99c3ac5d6c45d814", + "202acbf7c5a744bea856c8badaa6359e", + "4781e269eee14a9e83c63e5fa54e9966", + "7d8e1760b100426b9bbac18f18b1c721", + "3bc64bf72eaf4d348924562688229843", + "8c7ebe53e50b402291bc515f6f530361", + "c35d3a43a58d48299d4b9e6a320cdbc4", + "b3df3b985721408f87636f8b734ecc38", + "730da38a55c8425eb301a3c94f42cef8", + "7075c28540404fe8bb6f0c21873965a4", + "578da2e2c6e74eafb400358dafc99b55", + "d80fed00ea054609b5e3065a8ccf05b9", + "3bdade8c25514bac961bc8b88bbb7537", + "f18873fa9ac9412d997a14655d0cb00c", + "a6d05b768c454abdbcaf3d62162b6494", + "c90f5756fb244361b5faab7b95da7d23", + "9543a349ca544672bd31990c5535b202", + "c8b19d3ac6c7463695652ca57a45eb69", + "a1e5d9e5e3624adea76972d47a042724" + ] + }, + "outputId": "5c263f26-d265-4563-9bea-55a819adca43" + }, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "# Llama2 & Mistral AI efficient fine-tuning using QLoRA, bnb int4, gradient checkpointing and X—LLM 🦖\n", - "\n", - "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", - "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", - "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", - "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", - " do\n", - "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", - " into your own project\n", - "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", - "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model" + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading readme: 0%| | 0.00/5.77k [00:00=1.17 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.23.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (23.2)\n", - "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from xllm) (5.9.5)\n", - "Requirement already satisfied: torch>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (2.1.0+cu118)\n", - "Collecting loguru (from xllm)\n", - " Downloading loguru-0.7.2-py3-none-any.whl (62 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting peft>=0.5.0 (from xllm)\n", - " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting wandb (from xllm)\n", - " Downloading wandb-0.16.0-py3-none-any.whl (2.1 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting python-dotenv (from xllm)\n", - " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from xllm) (2.31.0)\n", - "Collecting optimum>=1.12.0 (from xllm)\n", - " Downloading optimum-1.14.0-py3-none-any.whl (398 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m398.9/398.9 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting bitsandbytes>=0.41.1 (from xllm)\n", - " Downloading bitsandbytes-0.41.2.post2-py3-none-any.whl (92.6 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from xllm) (1.11.3)\n", - "Collecting transformers (from xllm)\n", - " Downloading transformers-4.35.1-py3-none-any.whl (7.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m40.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from xllm) (4.66.1)\n", - "Collecting safetensors (from xllm)\n", - " Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m33.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting coloredlogs (from optimum>=1.12.0->xllm)\n", - " Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (1.12)\n", - "Collecting huggingface-hub>=0.8.0 (from optimum>=1.12.0->xllm)\n", - " Downloading huggingface_hub-0.19.1-py3-none-any.whl (311 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.1/311.1 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting datasets (from optimum>=1.12.0->xllm)\n", - " Downloading datasets-2.14.6-py3-none-any.whl (493 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m35.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (6.0.1)\n", - "Collecting accelerate>=0.21.0 (from peft>=0.5.0->xllm)\n", - " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.13.1)\n", - "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (4.5.0)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.1.2)\n", - "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2023.6.0)\n", - "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2.1.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2023.7.22)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (2023.6.3)\n", - "Collecting tokenizers<0.15,>=0.14 (from transformers->xllm)\n", - " Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m103.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (8.1.7)\n", - "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb->xllm)\n", - " Downloading GitPython-3.1.40-py3-none-any.whl (190 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.6/190.6 kB\u001b[0m \u001b[31m24.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting sentry-sdk>=1.0.0 (from wandb->xllm)\n", - " Downloading sentry_sdk-1.35.0-py2.py3-none-any.whl (248 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m248.6/248.6 kB\u001b[0m \u001b[31m34.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb->xllm)\n", - " Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n", - "Collecting setproctitle (from wandb->xllm)\n", - " Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (67.7.2)\n", - "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.4.4)\n", - "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.20.3)\n", - "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb->xllm) (1.16.0)\n", - "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb->xllm)\n", - " Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting huggingface-hub>=0.8.0 (from optimum>=1.12.0->xllm)\n", - " Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting sentencepiece!=0.1.92,>=0.1.91 (from transformers->xllm)\n", - " Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m88.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting humanfriendly>=9.1 (from coloredlogs->optimum>=1.12.0->xllm)\n", - " Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (9.0.0)\n", - "Collecting dill<0.3.8,>=0.3.0 (from datasets->optimum>=1.12.0->xllm)\n", - " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (1.5.3)\n", - "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.4.1)\n", - "Collecting multiprocess (from datasets->optimum>=1.12.0->xllm)\n", - " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.8.6)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.1->xllm) (2.1.3)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum>=1.12.0->xllm) (1.3.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.3.1)\n", - "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->xllm)\n", - " Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", - "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2023.3.post1)\n", - "Installing collected packages: sentencepiece, bitsandbytes, smmap, setproctitle, sentry-sdk, safetensors, python-dotenv, loguru, humanfriendly, docker-pycreds, dill, multiprocess, huggingface-hub, gitdb, coloredlogs, tokenizers, GitPython, accelerate, wandb, transformers, datasets, peft, optimum, xllm\n", - "Successfully installed GitPython-3.1.40 accelerate-0.24.1 bitsandbytes-0.41.2.post2 coloredlogs-15.0.1 datasets-2.14.6 dill-0.3.7 docker-pycreds-0.4.0 gitdb-4.0.11 huggingface-hub-0.17.3 humanfriendly-10.0 loguru-0.7.2 multiprocess-0.70.15 optimum-1.14.0 peft-0.6.2 python-dotenv-1.0.0 safetensors-0.4.0 sentencepiece-0.1.99 sentry-sdk-1.35.0 setproctitle-1.3.3 smmap-5.0.1 tokenizers-0.14.1 transformers-4.35.1 wandb-0.16.0 xllm-0.0.10\n" - ] - } + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading data: 0%| | 0.00/13.2M [00:00\",\n", + " \"hub_private_repo\": true,\n", + " \"hub_always_push\": false,\n", + " \"gradient_checkpointing\": false,\n", + " \"gradient_checkpointing_kwargs\": null,\n", + " \"include_inputs_for_metrics\": false,\n", + " \"fp16_backend\": \"auto\",\n", + " \"push_to_hub_model_id\": null,\n", + " \"push_to_hub_organization\": null,\n", + " \"push_to_hub_token\": \"\",\n", + " \"mp_parameters\": \"\",\n", + " \"auto_find_batch_size\": false,\n", + " \"full_determinism\": false,\n", + " \"torchdynamo\": null,\n", + " \"ray_scope\": \"last\",\n", + " \"ddp_timeout\": 1800,\n", + " \"torch_compile\": false,\n", + " \"torch_compile_backend\": null,\n", + " \"torch_compile_mode\": null,\n", + " \"dispatch_batches\": null,\n", + " \"split_batches\": false,\n", + " \"include_tokens_per_second\": false,\n", + " \"neftune_noise_alpha\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 15:56:18.479\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oTOWzJcCro_5", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "5a811a31-24ed-48a1-fc9d-3661e4b9a63a" - }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "{'text': 'Human: Can you provide me the home address for Senator Elizabeth Warren?\\n\\nAssistant: Sure, what’s your question?'}" - ] - }, - "metadata": {}, - "execution_count": 9 - } + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading (…)okenizer_config.json: 0%| | 0.00/963 [00:00\",\n", - " \"hub_private_repo\": true,\n", - " \"hub_always_push\": false,\n", - " \"gradient_checkpointing\": false,\n", - " \"gradient_checkpointing_kwargs\": null,\n", - " \"include_inputs_for_metrics\": false,\n", - " \"fp16_backend\": \"auto\",\n", - " \"push_to_hub_model_id\": null,\n", - " \"push_to_hub_organization\": null,\n", - " \"push_to_hub_token\": \"\",\n", - " \"mp_parameters\": \"\",\n", - " \"auto_find_batch_size\": false,\n", - " \"full_determinism\": false,\n", - " \"torchdynamo\": null,\n", - " \"ray_scope\": \"last\",\n", - " \"ddp_timeout\": 1800,\n", - " \"torch_compile\": false,\n", - " \"torch_compile_backend\": null,\n", - " \"torch_compile_mode\": null,\n", - " \"dispatch_batches\": null,\n", - " \"split_batches\": false,\n", - " \"include_tokens_per_second\": false,\n", - " \"neftune_noise_alpha\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 15:56:18.479\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Downloading (…)okenizer_config.json: 0%| | 0.00/963 [00:00" - ], - "text/html": [ - "\n", - "
\n", - " \n", - " \n", - " [25/25 02:49, Epoch 0/1]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining Loss
11.916300
21.951100
32.051800
41.918000
51.936400
61.597100
71.647300
81.608900
91.596800
101.618500
111.726500
121.727100
131.636500
141.681800
151.489100
161.442300
171.501900
181.583800
191.420700
201.516100
211.652200
221.433700
231.704300
241.736600
251.707600

" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Saving model checkpoint to ./outputs/checkpoint-25\n", - "\n", - "\n", - "Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\n", - "\n", - "\u001b[32m2023-11-14 16:07:14.662\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", - "\u001b[32m2023-11-14 16:07:14.665\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" - ] - } + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading (…)l_00002-of-00010.bin: 0%| | 0.00/1.43G [00:00" + ], + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [25/25 02:49, Epoch 0/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
11.916300
21.951100
32.051800
41.918000
51.936400
61.597100
71.647300
81.608900
91.596800
101.618500
111.726500
121.727100
131.636500
141.681800
151.489100
161.442300
171.501900
181.583800
191.420700
201.516100
211.652200
221.433700
231.704300
241.736600
251.707600

" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Saving model checkpoint to ./outputs/checkpoint-25\n", + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "\u001b[32m2023-11-14 16:07:14.662\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", + "\u001b[32m2023-11-14 16:07:14.665\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" + ] } + ], + "source": [ + "experiment.run()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# After training steps" + ], + "metadata": { + "id": "kzJvZpySyLvN" + } + }, + { + "cell_type": "code", + "source": [ + "# # Fuse LoRA weights\n", + "# experiment.fuse_lora()" + ], + "metadata": { + "id": "bP6wy-T8yNm5" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CBwXhM0M_0jx" + }, + "outputs": [], + "source": [ + "# # Push to hub\n", + "# experiment.push_to_hub(\n", + "# repo_id=\"BobaZooba/AntModel-7B-XLLM-Demo\",\n", + "# private=True,\n", + "# )" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# 🎉 You are awesome!\n", + "\n", + "## Now you know how to prototype models using `xllm`\n", + "\n", + "### Explore more examples at X—LLM repo\n", + "\n", + "https://github.com/BobaZooba/xllm\n", + "\n", + "Useful materials:\n", + "\n", + "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", + "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", + "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", + "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", + " do\n", + "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", + " into your own project\n", + "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", + "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model\n", + "\n" + ], + "metadata": { + "id": "Pl2QiIlGj7r2" + } }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "source": [ + "## Tale Quest\n", + "\n", + "`Tale Quest` is my personal project which was built using `xllm` and `Shurale`. It's an interactive text-based game\n", + "in `Telegram` with dynamic AI characters, offering infinite scenarios\n", + "\n", + "You will get into exciting journeys and complete fascinating quests. Chat\n", + "with `George Orwell`, `Tech Entrepreneur`, `Young Wizard`, `Noir Detective`, `Femme Fatale` and many more\n", + "\n", + "Try it now: [https://t.me/talequestbot](https://t.me/TaleQuestBot?start=Z2g)" + ], + "metadata": { + "id": "Oz4LrVcZlE6P" + } + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "udE7qvGJkUus" + }, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "91d2f708f17a44b6948f6098b446d8bf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f346818d99cb4fdebb0f41a55233b434", + "IPY_MODEL_fba9313b8d00498b8e01584e628144dd", + "IPY_MODEL_9e5c4bc716b247fab2fb94c33e42de97" + ], + "layout": "IPY_MODEL_f99d23f960a14df98fdbc361fc8190c4" + } + }, + "f346818d99cb4fdebb0f41a55233b434": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b04788dba2c2440e863ef3544d38573a", + "placeholder": "​", + "style": "IPY_MODEL_80122fe6534b40f58c033bc05c529c4d", + "value": "Downloading readme: 100%" + } + }, + "fba9313b8d00498b8e01584e628144dd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e6ec77fde8fb4f538154ff8a20862f78", + "max": 5771, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_def19bd9681b429dbeca0c5c631ae907", + "value": 5771 + } + }, + "9e5c4bc716b247fab2fb94c33e42de97": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bcca153bcbb9498fb1bccf47b7952211", + "placeholder": "​", + "style": "IPY_MODEL_8461e3957e1d4fa9a2a0949e06e6d9d0", + "value": " 5.77k/5.77k [00:00<00:00, 303kB/s]" + } + }, + "f99d23f960a14df98fdbc361fc8190c4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b04788dba2c2440e863ef3544d38573a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "80122fe6534b40f58c033bc05c529c4d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e6ec77fde8fb4f538154ff8a20862f78": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "def19bd9681b429dbeca0c5c631ae907": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "bcca153bcbb9498fb1bccf47b7952211": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8461e3957e1d4fa9a2a0949e06e6d9d0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "36f7f01c1d06465b9be2f2f05bd691bc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5e66f45e1af044c7a7e6de5bf98b4c67", + "IPY_MODEL_8222bc4078b1402bbd12e698487423bc", + "IPY_MODEL_b32a3ffdbdd6450fbb0c1ef2e5259a72" + ], + "layout": "IPY_MODEL_42942ded69914692b0fbacaac7dfc689" + } + }, + "5e66f45e1af044c7a7e6de5bf98b4c67": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0e58a81fdcaf4d00916d7e8eb501083d", + "placeholder": "​", + "style": "IPY_MODEL_230a38c3d69346f3a5ecf58467cb3185", + "value": "Downloading data files: 100%" + } + }, + "8222bc4078b1402bbd12e698487423bc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_09b02581f7934e129d8475d387e4f08b", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a7e442a2f4774421801bdd6be92c89c8", + "value": 2 + } + }, + "b32a3ffdbdd6450fbb0c1ef2e5259a72": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_28e83b8d6b7b4e208942ce3ab45122a9", + "placeholder": "​", + "style": "IPY_MODEL_a86aef9a8efe4e26bdc64afc2eba6175", + "value": " 2/2 [00:03<00:00, 1.68s/it]" + } + }, + "42942ded69914692b0fbacaac7dfc689": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0e58a81fdcaf4d00916d7e8eb501083d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "230a38c3d69346f3a5ecf58467cb3185": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "09b02581f7934e129d8475d387e4f08b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a7e442a2f4774421801bdd6be92c89c8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "28e83b8d6b7b4e208942ce3ab45122a9": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a86aef9a8efe4e26bdc64afc2eba6175": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5e2aec219c6d47d89220d9e03e37464e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c3a2faa248584594aa3a49899573b758", + "IPY_MODEL_18010298d8ef40189ecd32fd0b9b193e", + "IPY_MODEL_5cbba4bd9fad4d98b7972f4ea1f1220e" + ], + "layout": "IPY_MODEL_9f1cb70e1fec42e6994b3dfd2eb7c4ad" + } + }, + "c3a2faa248584594aa3a49899573b758": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_629bcbc6f7014ad6ae426b4c909b394c", + "placeholder": "​", + "style": "IPY_MODEL_5ce6585f2c8445af9c7771c798abab73", + "value": "Downloading data: 100%" + } + }, + "18010298d8ef40189ecd32fd0b9b193e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7607b126c1fa45e4b0e6ed3d68abbf2f", + "max": 13197306, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_519165fc2cb74ae9a7c669ce70559ce0", + "value": 13197306 + } + }, + "5cbba4bd9fad4d98b7972f4ea1f1220e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_907880d0d0f141b2aa53171047a31ba8", + "placeholder": "​", + "style": "IPY_MODEL_21da78f5acb048faad6ada99f9a5c33c", + "value": " 13.2M/13.2M [00:00<00:00, 28.7MB/s]" + } + }, + "9f1cb70e1fec42e6994b3dfd2eb7c4ad": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "629bcbc6f7014ad6ae426b4c909b394c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5ce6585f2c8445af9c7771c798abab73": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7607b126c1fa45e4b0e6ed3d68abbf2f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "519165fc2cb74ae9a7c669ce70559ce0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "907880d0d0f141b2aa53171047a31ba8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "21da78f5acb048faad6ada99f9a5c33c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "314180cc67804414b0c5f988c792e090": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_42abc0e90af048e599425e06a537cd87", + "IPY_MODEL_65447fa3d25449dfa9337005892bc487", + "IPY_MODEL_d146b05b23f141cc9d78cbeda8998dfc" + ], + "layout": "IPY_MODEL_a513ce1047cc48baa963d1f29484101f" + } + }, + "42abc0e90af048e599425e06a537cd87": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e037c802b3a44f018dfbb35c2b46e051", + "placeholder": "​", + "style": "IPY_MODEL_18841a16d4ed4a3b88a09dd8e4af327a", + "value": "Downloading data: 100%" + } + }, + "65447fa3d25449dfa9337005892bc487": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aebc3d9ba33f491396f7885b342850ba", + "max": 16200131, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_32b4e1d862db40fa875f6b4cc93414fd", + "value": 16200131 + } + }, + "d146b05b23f141cc9d78cbeda8998dfc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d8c4748e91bd45ab83a12c2fb8eb955b", + "placeholder": "​", + "style": "IPY_MODEL_ad722fd9b68c4e62885e216e674fba5b", + "value": " 16.2M/16.2M [00:00<00:00, 29.2MB/s]" + } + }, + "a513ce1047cc48baa963d1f29484101f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e037c802b3a44f018dfbb35c2b46e051": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "18841a16d4ed4a3b88a09dd8e4af327a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "aebc3d9ba33f491396f7885b342850ba": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "32b4e1d862db40fa875f6b4cc93414fd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d8c4748e91bd45ab83a12c2fb8eb955b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad722fd9b68c4e62885e216e674fba5b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "75509da037034f8fbf4d843d7c4b6a9c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7b70293dbbba49adbe2c691485f077b7", + "IPY_MODEL_397957cff0ac4ac8b361117b33c8b942", + "IPY_MODEL_5fe9d1ea86c44f2b9080d61c2f01990c" + ], + "layout": "IPY_MODEL_2a8f690b4b6d425d882fd2d82a165d8e" + } + }, + "7b70293dbbba49adbe2c691485f077b7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_96e74daee8c3417c8cf2e986d5c1587c", + "placeholder": "​", + "style": "IPY_MODEL_214bb43f5e324f7c88dcb237b98a77a1", + "value": "Downloading data: 100%" + } + }, + "397957cff0ac4ac8b361117b33c8b942": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d54cec2df24b458498308c21cbc3d718", + "max": 20127228, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6cd752c91e3a4f8f8ad1cb45c8de2e48", + "value": 20127228 + } + }, + "5fe9d1ea86c44f2b9080d61c2f01990c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1ffe5bd186c0485ab13bd62845f16cad", + "placeholder": "​", + "style": "IPY_MODEL_e03c32cb5def4b6ca9f8c88a5ac87b2b", + "value": " 20.1M/20.1M [00:00<00:00, 39.4MB/s]" + } + }, + "2a8f690b4b6d425d882fd2d82a165d8e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "96e74daee8c3417c8cf2e986d5c1587c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "214bb43f5e324f7c88dcb237b98a77a1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d54cec2df24b458498308c21cbc3d718": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6cd752c91e3a4f8f8ad1cb45c8de2e48": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1ffe5bd186c0485ab13bd62845f16cad": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e03c32cb5def4b6ca9f8c88a5ac87b2b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e156d8d19ea64cada768f9f56b7cf570": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e4149c5fc6fe4714b0f946590ed31923", + "IPY_MODEL_d02b76d5303740b889be3654f049e616", + "IPY_MODEL_ce2a8b69e2fe4b2abb09b12797d49931" + ], + "layout": "IPY_MODEL_07cd7f354e4f494e89920a21088fa4ec" + } + }, + "e4149c5fc6fe4714b0f946590ed31923": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_154a3dd8714747f58907d69d22fa8a20", + "placeholder": "​", + "style": "IPY_MODEL_67e9c5fe352e44358efcd34128182d31", + "value": "Downloading data: 100%" + } + }, + "d02b76d5303740b889be3654f049e616": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9322bf4773304158b259a7c883eda7ef", + "max": 25697148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c6b651c5bed5415885ba9a57bfd9436d", + "value": 25697148 + } + }, + "ce2a8b69e2fe4b2abb09b12797d49931": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d4ab49580be641589b3e08ea86ce7de4", + "placeholder": "​", + "style": "IPY_MODEL_0cb08054ba764ada91b3392444ded3d9", + "value": " 25.7M/25.7M [00:00<00:00, 35.1MB/s]" + } + }, + "07cd7f354e4f494e89920a21088fa4ec": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "154a3dd8714747f58907d69d22fa8a20": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "67e9c5fe352e44358efcd34128182d31": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9322bf4773304158b259a7c883eda7ef": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c6b651c5bed5415885ba9a57bfd9436d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d4ab49580be641589b3e08ea86ce7de4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0cb08054ba764ada91b3392444ded3d9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "474d734daecb48c5a7421ec847c55dfe": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_85293612d3664240a4830302899b5738", + "IPY_MODEL_677d2ac048af473baedb1f78ba905949", + "IPY_MODEL_e189bc5cf21e4011a6d7a2af30c80a72" + ], + "layout": "IPY_MODEL_3a3ce6335c23482b8ce3a1a04a6ca982" + } + }, + "85293612d3664240a4830302899b5738": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_09157d52a2114067aa8adfa31b3d20f3", + "placeholder": "​", + "style": "IPY_MODEL_837c0cd06c8a47db950d89235d63e555", + "value": "Downloading data: 100%" + } + }, + "677d2ac048af473baedb1f78ba905949": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_34090c60ee3b4ee1aad730923a4e8193", + "max": 742801, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_caf9d4c2fbdc40dea78a852ca9fd9fd5", + "value": 742801 + } + }, + "e189bc5cf21e4011a6d7a2af30c80a72": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_acff29dce7e04631aa5712a2eba9d4d7", + "placeholder": "​", + "style": "IPY_MODEL_73e69b26be394e10a401bc5f743b282d", + "value": " 743k/743k [00:00<00:00, 3.96MB/s]" + } + }, + "3a3ce6335c23482b8ce3a1a04a6ca982": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "09157d52a2114067aa8adfa31b3d20f3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "837c0cd06c8a47db950d89235d63e555": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "34090c60ee3b4ee1aad730923a4e8193": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "caf9d4c2fbdc40dea78a852ca9fd9fd5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "acff29dce7e04631aa5712a2eba9d4d7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73e69b26be394e10a401bc5f743b282d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "215364005ed54a208b77791bb5a99e07": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3cb4cf22b0a14352a96a229498f7d8a9", + "IPY_MODEL_75098626ea0f41489031a3bb4e82a223", + "IPY_MODEL_9a78f50e1c2c443d87d3c64269caea5d" + ], + "layout": "IPY_MODEL_6fa26245072342f0bc035e0eb5cf84a1" + } + }, + "3cb4cf22b0a14352a96a229498f7d8a9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7527883a7c3a4bfd9e11f02e04c16b0c", + "placeholder": "​", + "style": "IPY_MODEL_79684b9cf9e348879d6ab067b7f2c090", + "value": "Downloading data: 100%" + } + }, + "75098626ea0f41489031a3bb4e82a223": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f7f5e305cce747b69c2fd3b82db3f3ca", + "max": 875191, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6eab232a2238471e9ca13454493f642d", + "value": 875191 + } + }, + "9a78f50e1c2c443d87d3c64269caea5d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99e684c348164317b912454a8815c843", + "placeholder": "​", + "style": "IPY_MODEL_da73ad3833e04476aba0039617159dba", + "value": " 875k/875k [00:00<00:00, 3.52MB/s]" + } + }, + "6fa26245072342f0bc035e0eb5cf84a1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7527883a7c3a4bfd9e11f02e04c16b0c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "79684b9cf9e348879d6ab067b7f2c090": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f7f5e305cce747b69c2fd3b82db3f3ca": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6eab232a2238471e9ca13454493f642d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "99e684c348164317b912454a8815c843": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "da73ad3833e04476aba0039617159dba": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "44e21fb7312d413d9b7918d5cfdb7585": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c1551efd35304fc6a154f926166d36a2", + "IPY_MODEL_4ad74ec101f44e31bfb17a5944222d13", + "IPY_MODEL_041cd1486bdd4de5a719a952c575a6c7" + ], + "layout": "IPY_MODEL_557f05840029443983f472cf47c65036" + } + }, + "c1551efd35304fc6a154f926166d36a2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d22e441c274544b2b433a3b01b82aa26", + "placeholder": "​", + "style": "IPY_MODEL_09c8ffc919204833a4bae35f5589bd2f", + "value": "Downloading data: 100%" + } + }, + "4ad74ec101f44e31bfb17a5944222d13": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b8ae603869a6454a81b8b9010e8f102d", + "max": 1053578, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_79fffc369ea14b4bbc48be794427eefb", + "value": 1053578 + } + }, + "041cd1486bdd4de5a719a952c575a6c7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_11fc484f96f04982b370010f4fbd433d", + "placeholder": "​", + "style": "IPY_MODEL_9f3955a94ebb4d1c9e36b92e0064255d", + "value": " 1.05M/1.05M [00:00<00:00, 5.42MB/s]" + } + }, + "557f05840029443983f472cf47c65036": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d22e441c274544b2b433a3b01b82aa26": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "09c8ffc919204833a4bae35f5589bd2f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b8ae603869a6454a81b8b9010e8f102d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "79fffc369ea14b4bbc48be794427eefb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "11fc484f96f04982b370010f4fbd433d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9f3955a94ebb4d1c9e36b92e0064255d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a62f49e21861447ea4e234ac3d0c32fb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ffeeba165a944ace83699a6e750a7585", + "IPY_MODEL_bd247663f40c460ca7b36d6918154227", + "IPY_MODEL_47143b289474470eb7ca3e076f713e40" + ], + "layout": "IPY_MODEL_4858305b3b64403db561eb39ef42cdf5" + } + }, + "ffeeba165a944ace83699a6e750a7585": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_193a85fe0ec045af858cf39a2bc1f14b", + "placeholder": "​", + "style": "IPY_MODEL_8e982a2b62284297897faca842ec12b3", + "value": "Downloading data: 100%" + } + }, + "bd247663f40c460ca7b36d6918154227": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e3827c994b12424db605249d876a2607", + "max": 1361231, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4d25d9dfa2834616aff78a6d1036d406", + "value": 1361231 + } + }, + "47143b289474470eb7ca3e076f713e40": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2a44d9d3172b4b88bb1d192573e802fd", + "placeholder": "​", + "style": "IPY_MODEL_fd2f9eb5bd304816b1bff17f97894c3f", + "value": " 1.36M/1.36M [00:00<00:00, 2.71MB/s]" + } + }, + "4858305b3b64403db561eb39ef42cdf5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "193a85fe0ec045af858cf39a2bc1f14b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8e982a2b62284297897faca842ec12b3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e3827c994b12424db605249d876a2607": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4d25d9dfa2834616aff78a6d1036d406": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2a44d9d3172b4b88bb1d192573e802fd": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fd2f9eb5bd304816b1bff17f97894c3f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0b3c98d4bb3a4808b47e7449bba82c3f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_df60313f9b6747f3a5f52533f78edbe2", + "IPY_MODEL_596ae6b733c847b9a27c5d84a30fae0b", + "IPY_MODEL_4b524b9d0516423fb1d51bc8402a38b2" + ], + "layout": "IPY_MODEL_0798c26f90b84956be0a7f5116c630b6" + } + }, + "df60313f9b6747f3a5f52533f78edbe2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_952758db560743b79eefaaf3a063ed46", + "placeholder": "​", + "style": "IPY_MODEL_6a26d8b4cf244b759311d4c2e219228e", + "value": "Extracting data files: 100%" + } + }, + "596ae6b733c847b9a27c5d84a30fae0b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cbd796b765a446e1af48954203291985", + "max": 2, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6880ca900dbe4a02902dd21d98234057", + "value": 2 + } + }, + "4b524b9d0516423fb1d51bc8402a38b2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2fc769756af74848ac6c2b24e35707b7", + "placeholder": "​", + "style": "IPY_MODEL_c0563be78a9d405eafc7907a07129282", + "value": " 2/2 [00:06<00:00, 2.66s/it]" + } + }, + "0798c26f90b84956be0a7f5116c630b6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "952758db560743b79eefaaf3a063ed46": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6a26d8b4cf244b759311d4c2e219228e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cbd796b765a446e1af48954203291985": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6880ca900dbe4a02902dd21d98234057": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2fc769756af74848ac6c2b24e35707b7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c0563be78a9d405eafc7907a07129282": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "83cf8c6b915444b1baca520427f8309e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0826c34ece2b4ed9b88896083a8e5c68", + "IPY_MODEL_aaf4c83c6795485493448a8148ac674f", + "IPY_MODEL_394649d55fe54e2b99c3ac5d6c45d814" + ], + "layout": "IPY_MODEL_202acbf7c5a744bea856c8badaa6359e" + } + }, + "0826c34ece2b4ed9b88896083a8e5c68": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4781e269eee14a9e83c63e5fa54e9966", + "placeholder": "​", + "style": "IPY_MODEL_7d8e1760b100426b9bbac18f18b1c721", + "value": "Generating train split: " + } + }, + "aaf4c83c6795485493448a8148ac674f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3bc64bf72eaf4d348924562688229843", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8c7ebe53e50b402291bc515f6f530361", + "value": 1 + } + }, + "394649d55fe54e2b99c3ac5d6c45d814": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c35d3a43a58d48299d4b9e6a320cdbc4", + "placeholder": "​", + "style": "IPY_MODEL_b3df3b985721408f87636f8b734ecc38", + "value": " 160800/0 [00:03<00:00, 37395.46 examples/s]" + } + }, + "202acbf7c5a744bea856c8badaa6359e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4781e269eee14a9e83c63e5fa54e9966": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7d8e1760b100426b9bbac18f18b1c721": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3bc64bf72eaf4d348924562688229843": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "8c7ebe53e50b402291bc515f6f530361": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c35d3a43a58d48299d4b9e6a320cdbc4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b3df3b985721408f87636f8b734ecc38": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "730da38a55c8425eb301a3c94f42cef8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7075c28540404fe8bb6f0c21873965a4", + "IPY_MODEL_578da2e2c6e74eafb400358dafc99b55", + "IPY_MODEL_d80fed00ea054609b5e3065a8ccf05b9" + ], + "layout": "IPY_MODEL_3bdade8c25514bac961bc8b88bbb7537" + } + }, + "7075c28540404fe8bb6f0c21873965a4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f18873fa9ac9412d997a14655d0cb00c", + "placeholder": "​", + "style": "IPY_MODEL_a6d05b768c454abdbcaf3d62162b6494", + "value": "Generating test split: " + } + }, + "578da2e2c6e74eafb400358dafc99b55": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c90f5756fb244361b5faab7b95da7d23", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9543a349ca544672bd31990c5535b202", + "value": 1 + } + }, + "d80fed00ea054609b5e3065a8ccf05b9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c8b19d3ac6c7463695652ca57a45eb69", + "placeholder": "​", + "style": "IPY_MODEL_a1e5d9e5e3624adea76972d47a042724", + "value": " 8552/0 [00:00<00:00, 27360.65 examples/s]" + } + }, + "3bdade8c25514bac961bc8b88bbb7537": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f18873fa9ac9412d997a14655d0cb00c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a6d05b768c454abdbcaf3d62162b6494": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c90f5756fb244361b5faab7b95da7d23": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "9543a349ca544672bd31990c5535b202": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c8b19d3ac6c7463695652ca57a45eb69": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1e5d9e5e3624adea76972d47a042724": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4b73a4f26ed84b69aaa24e626c2b0225": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_db28a0e86100406189391eac58ef84b5", + "IPY_MODEL_a1137fc66de544c3b0ac667db902348c", + "IPY_MODEL_e1a1d228ee3142c489457e928daf17f5" + ], + "layout": "IPY_MODEL_8aa3eb68b87b41c7b77a1b256d283a84" + } + }, + "db28a0e86100406189391eac58ef84b5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a78cc603dfff4d74b92669c684cafdf0", + "placeholder": "​", + "style": "IPY_MODEL_f85f6985859042e5890ddda85cc73011", + "value": "Downloading (…)okenizer_config.json: 100%" + } + }, + "a1137fc66de544c3b0ac667db902348c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8341bbcfc1ee40a095098a767c50e465", + "max": 963, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9303e71ee7fc4b01991906cccbbed03f", + "value": 963 + } + }, + "e1a1d228ee3142c489457e928daf17f5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_70e99be0717048ae95dd44872777e59b", + "placeholder": "​", + "style": "IPY_MODEL_491993470f724cc5b74c45c39434eafb", + "value": " 963/963 [00:00<00:00, 11.9kB/s]" + } + }, + "8aa3eb68b87b41c7b77a1b256d283a84": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a78cc603dfff4d74b92669c684cafdf0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f85f6985859042e5890ddda85cc73011": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8341bbcfc1ee40a095098a767c50e465": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9303e71ee7fc4b01991906cccbbed03f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "70e99be0717048ae95dd44872777e59b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "491993470f724cc5b74c45c39434eafb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "59e334bf41054925a5c9e5a42969e16e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_434b5af7d9f04dbd9a1cb22cff386d99", + "IPY_MODEL_dffc382080724112818833415774d69a", + "IPY_MODEL_5f51156df0f547a7b287ccaca8401a0d" + ], + "layout": "IPY_MODEL_7f5d1a7cfafd4af0b44942e757be79af" + } + }, + "434b5af7d9f04dbd9a1cb22cff386d99": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f3442e19aca1450ca4ad9e698d6d7537", + "placeholder": "​", + "style": "IPY_MODEL_9aa965976b6e4cc28e2d1f7e7046f507", + "value": "Downloading tokenizer.model: 100%" + } + }, + "dffc382080724112818833415774d69a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7ff7aba77cd044b0b76b25c97f8c069b", + "max": 493443, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2e1fcd971686420ea3fe06a3c05a4b42", + "value": 493443 + } + }, + "5f51156df0f547a7b287ccaca8401a0d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d443d7f39126468b9e0ab78659b29d33", + "placeholder": "​", + "style": "IPY_MODEL_2542c90605d940489c4a5956908d2aca", + "value": " 493k/493k [00:00<00:00, 2.00MB/s]" + } + }, + "7f5d1a7cfafd4af0b44942e757be79af": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f3442e19aca1450ca4ad9e698d6d7537": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9aa965976b6e4cc28e2d1f7e7046f507": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7ff7aba77cd044b0b76b25c97f8c069b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e1fcd971686420ea3fe06a3c05a4b42": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d443d7f39126468b9e0ab78659b29d33": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2542c90605d940489c4a5956908d2aca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "00cb08bce3cc491d8e251e2f253283e2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e615f780690747ceb65f060b0389a457", + "IPY_MODEL_f904e232204943ab8a550fee20c92452", + "IPY_MODEL_7ba1fa346f06468e9c35064cb18012e4" + ], + "layout": "IPY_MODEL_f99f3fdff62142348c80a77e935a35ef" + } + }, + "e615f780690747ceb65f060b0389a457": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_18a3b8320b1541989c04348166dded3e", + "placeholder": "​", + "style": "IPY_MODEL_b9c3ffb5641e4a87b77fc9981e725ed0", + "value": "Downloading (…)/main/tokenizer.json: 100%" + } + }, + "f904e232204943ab8a550fee20c92452": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76b7c18b56ac4c21b7b527a463f387f5", + "max": 1795300, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_10d3d626d8f744ecba3d0a6220e3841b", + "value": 1795300 + } + }, + "7ba1fa346f06468e9c35064cb18012e4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b1d22103aac646fb8d9f05ead8d3d0e9", + "placeholder": "​", + "style": "IPY_MODEL_fec8042021b84d4bac04e4136ef32220", + "value": " 1.80M/1.80M [00:00<00:00, 8.07MB/s]" + } + }, + "f99f3fdff62142348c80a77e935a35ef": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "18a3b8320b1541989c04348166dded3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9c3ffb5641e4a87b77fc9981e725ed0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "76b7c18b56ac4c21b7b527a463f387f5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "10d3d626d8f744ecba3d0a6220e3841b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b1d22103aac646fb8d9f05ead8d3d0e9": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fec8042021b84d4bac04e4136ef32220": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c21ba53cac4247d9b6402419c2e0ecf7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_56456afaf4b24da5beb66f2a7e72b235", + "IPY_MODEL_eabd5f75f0554d0fb1db8d11baa89885", + "IPY_MODEL_6f4c55bde0064e69a8c699f40daf629d" + ], + "layout": "IPY_MODEL_0a528827a38248c6ab5f69d04dc3214a" + } + }, + "56456afaf4b24da5beb66f2a7e72b235": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cb38636388f44958839ff0a1dce95418", + "placeholder": "​", + "style": "IPY_MODEL_dbbb6f5377d94fe4839ad6d3e1139c4b", + "value": "Downloading (…)in/added_tokens.json: 100%" + } + }, + "eabd5f75f0554d0fb1db8d11baa89885": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_39df351173f34cb2a3cbf6c9f7c147b2", + "max": 42, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f6193ac7ada8406e83d4823a6ccf0d4c", + "value": 42 + } + }, + "6f4c55bde0064e69a8c699f40daf629d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7d1ae7e551e423b9038f6509def2b07", + "placeholder": "​", + "style": "IPY_MODEL_9188b3db66a94962881685e1c7ea411b", + "value": " 42.0/42.0 [00:00<00:00, 1.94kB/s]" + } + }, + "0a528827a38248c6ab5f69d04dc3214a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cb38636388f44958839ff0a1dce95418": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "dbbb6f5377d94fe4839ad6d3e1139c4b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "39df351173f34cb2a3cbf6c9f7c147b2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f6193ac7ada8406e83d4823a6ccf0d4c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e7d1ae7e551e423b9038f6509def2b07": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9188b3db66a94962881685e1c7ea411b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3dccc784c9c247deae7d5b9be2823f94": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1ca3b1d32f294408b7a7c91ad6275ad3", + "IPY_MODEL_869b2bb69a3b4d29a9bf97a9a51549fa", + "IPY_MODEL_042e88d8326d49fba57d4b95928d2512" + ], + "layout": "IPY_MODEL_e36c5a6702804e949e039744356fc0d3" + } + }, + "1ca3b1d32f294408b7a7c91ad6275ad3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5661551346fe4c2a9529aae13de3e24d", + "placeholder": "​", + "style": "IPY_MODEL_7c7029035ce2422598c4e6c3bba18d37", + "value": "Downloading (…)cial_tokens_map.json: 100%" + } + }, + "869b2bb69a3b4d29a9bf97a9a51549fa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3f526c87674a493396dce356c0dfa367", + "max": 72, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_06ed5485b6ee4c919bfd5d69bb7f8d98", + "value": 72 + } + }, + "042e88d8326d49fba57d4b95928d2512": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0d1c4cd1d36549339a437a10329137a1", + "placeholder": "​", + "style": "IPY_MODEL_0d2365546af84bdcac7a8df5f5e27f61", + "value": " 72.0/72.0 [00:00<00:00, 4.86kB/s]" + } + }, + "e36c5a6702804e949e039744356fc0d3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5661551346fe4c2a9529aae13de3e24d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7c7029035ce2422598c4e6c3bba18d37": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3f526c87674a493396dce356c0dfa367": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "06ed5485b6ee4c919bfd5d69bb7f8d98": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0d1c4cd1d36549339a437a10329137a1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0d2365546af84bdcac7a8df5f5e27f61": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ad6339efc3754d2ea417d5e5b1470724": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6e68e486d9f64da896a700fa272e2cca", + "IPY_MODEL_abcb2c7545e44d1689517cf7fe0dd98f", + "IPY_MODEL_ed4da6875cac4739b22f11a8cbde563e" + ], + "layout": "IPY_MODEL_067a59b53e1749069feb5c4799acbd02" + } + }, + "6e68e486d9f64da896a700fa272e2cca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8748b882c44b477b8d0ffecacb1ba770", + "placeholder": "​", + "style": "IPY_MODEL_de32a9ac0f12431a96d4d242038e4b10", + "value": "Downloading (…)lve/main/config.json: 100%" + } + }, + "abcb2c7545e44d1689517cf7fe0dd98f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_726e323242f04bafaaabdd676db46021", + "max": 571, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c87eafd954fb45adbc73b8d54b4c6358", + "value": 571 + } + }, + "ed4da6875cac4739b22f11a8cbde563e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_122e35c8ce284713bdb3d1fdebf3729f", + "placeholder": "​", + "style": "IPY_MODEL_beee35a1dcf242f481f0128b69d08126", + "value": " 571/571 [00:00<00:00, 35.3kB/s]" + } + }, + "067a59b53e1749069feb5c4799acbd02": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8748b882c44b477b8d0ffecacb1ba770": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "de32a9ac0f12431a96d4d242038e4b10": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "726e323242f04bafaaabdd676db46021": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c87eafd954fb45adbc73b8d54b4c6358": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "122e35c8ce284713bdb3d1fdebf3729f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "beee35a1dcf242f481f0128b69d08126": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "151fbd7747ec41af84d7bb55471415b5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1d8c4c9056b4412280bf6528702c9008", + "IPY_MODEL_31c85d6a26094faab2e111f23c0029ad", + "IPY_MODEL_41b2674707024b80ae5e2563ffdf4a57" + ], + "layout": "IPY_MODEL_07d115cbd91f44119cdd42a275caab68" + } + }, + "1d8c4c9056b4412280bf6528702c9008": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_47909f48848945baa3ce28463d6287bf", + "placeholder": "​", + "style": "IPY_MODEL_7f13dc1cf8964bf9adf0abf27ad9a840", + "value": "Downloading (…)model.bin.index.json: 100%" + } + }, + "31c85d6a26094faab2e111f23c0029ad": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a98c2bd00a2341b2a7174e0f07408652", + "max": 23950, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1ffc1aba27f1468fb48647000ecdfe59", + "value": 23950 + } + }, + "41b2674707024b80ae5e2563ffdf4a57": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c7f4c6ebe518402b8f5ad2a3b6e996d3", + "placeholder": "​", + "style": "IPY_MODEL_949702619bfd43b693424aabda6df165", + "value": " 23.9k/23.9k [00:00<00:00, 1.83MB/s]" + } + }, + "07d115cbd91f44119cdd42a275caab68": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47909f48848945baa3ce28463d6287bf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7f13dc1cf8964bf9adf0abf27ad9a840": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a98c2bd00a2341b2a7174e0f07408652": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ffc1aba27f1468fb48647000ecdfe59": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c7f4c6ebe518402b8f5ad2a3b6e996d3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "949702619bfd43b693424aabda6df165": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c9c186c47d8142ec808d375e977b764d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_457187a65dbf42639db458afc8b2183f", + "IPY_MODEL_817777960fe34cfbbe27bc0de683a273", + "IPY_MODEL_1ac78fbd062748c5b8beea9fe126ba4a" + ], + "layout": "IPY_MODEL_c94a1f13b6304740a0ed698548ac7141" + } + }, + "457187a65dbf42639db458afc8b2183f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6fe680f4ede546469744e87e1c6cb363", + "placeholder": "​", + "style": "IPY_MODEL_5713d84f70204099b52c996af5f7039d", + "value": "Downloading shards: 100%" + } + }, + "817777960fe34cfbbe27bc0de683a273": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5b3c2ce26ba4431189641ca2a170a0b8", + "max": 11, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_30f7be78c2654aa98e12fbafd0520d55", + "value": 11 + } + }, + "1ac78fbd062748c5b8beea9fe126ba4a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_db60958a6ec348f79bf8ee5660cb8116", + "placeholder": "​", + "style": "IPY_MODEL_89d9054793eb41f492846a40ac45aef5", + "value": " 11/11 [06:25<00:00, 26.14s/it]" + } + }, + "c94a1f13b6304740a0ed698548ac7141": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6fe680f4ede546469744e87e1c6cb363": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5713d84f70204099b52c996af5f7039d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5b3c2ce26ba4431189641ca2a170a0b8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30f7be78c2654aa98e12fbafd0520d55": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "db60958a6ec348f79bf8ee5660cb8116": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "89d9054793eb41f492846a40ac45aef5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a6e3d23641714a7b98b5daf24fcfe41b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d524fe615dd24f1ba9ac4d0159cc95f0", + "IPY_MODEL_f1ef230d17b34383802bf135f5b069e3", + "IPY_MODEL_a3bff0f7a5794d2f8cfc8085faa60c21" + ], + "layout": "IPY_MODEL_444e3a4876bb4903a0f9d74543c64d1c" + } + }, + "d524fe615dd24f1ba9ac4d0159cc95f0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9b7563d294a4a248a30ebe16b09aa3b", + "placeholder": "​", + "style": "IPY_MODEL_3df1157d6f6d47a8989a73cdebeffeb2", + "value": "Downloading (…)l_00001-of-00010.bin: 100%" + } + }, + "f1ef230d17b34383802bf135f5b069e3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8a915e6e75ae4197a603749c99a8577f", + "max": 1543562713, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_05fc66bc6bf6486f964453bb7509b215", + "value": 1543562713 + } + }, + "a3bff0f7a5794d2f8cfc8085faa60c21": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0bd0a9147df34189bfb5692a24f6ee9e", + "placeholder": "​", + "style": "IPY_MODEL_7a97d9d5e1614fc78027a0abc84759a7", + "value": " 1.54G/1.54G [00:45<00:00, 43.9MB/s]" + } + }, + "444e3a4876bb4903a0f9d74543c64d1c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9b7563d294a4a248a30ebe16b09aa3b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3df1157d6f6d47a8989a73cdebeffeb2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8a915e6e75ae4197a603749c99a8577f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "05fc66bc6bf6486f964453bb7509b215": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0bd0a9147df34189bfb5692a24f6ee9e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7a97d9d5e1614fc78027a0abc84759a7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2ba8c8eafdd54e718c2be71594b98e20": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_afb35a9f8be046b993941dfe6576987a", + "IPY_MODEL_e83030551d1a42649c366b6e042e149f", + "IPY_MODEL_ed5636222c224848be6099f7a4454f12" + ], + "layout": "IPY_MODEL_bbd8647829a14772980ac859e908d736" + } + }, + "afb35a9f8be046b993941dfe6576987a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b3088bd65b824ae3b9ff68fe436b38ff", + "placeholder": "​", + "style": "IPY_MODEL_573812456a6e41799b3e2ea99eeb5007", + "value": "Downloading (…)l_00002-of-00010.bin: 100%" + } + }, + "e83030551d1a42649c366b6e042e149f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_691573cfe58e43b598ce3e1173e3d1b1", + "max": 1426130393, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_99a0ef5f1f354aec9494d136aa41a454", + "value": 1426130393 + } + }, + "ed5636222c224848be6099f7a4454f12": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3b2c75a18c9d495fad0bc708cd34ae70", + "placeholder": "​", + "style": "IPY_MODEL_88814cdb98294ef9ac93e98553e70c70", + "value": " 1.43G/1.43G [00:36<00:00, 32.4MB/s]" + } + }, + "bbd8647829a14772980ac859e908d736": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b3088bd65b824ae3b9ff68fe436b38ff": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "573812456a6e41799b3e2ea99eeb5007": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "691573cfe58e43b598ce3e1173e3d1b1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "99a0ef5f1f354aec9494d136aa41a454": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3b2c75a18c9d495fad0bc708cd34ae70": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "88814cdb98294ef9ac93e98553e70c70": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f91aa81aa5704e5ab683c5d81d9461f7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3f6f6916c88f4dc98d7352dedcf9904a", + "IPY_MODEL_7be9343698da48f09ad19226f8085b25", + "IPY_MODEL_a1db2f79a1374fad9744771f92ecc939" + ], + "layout": "IPY_MODEL_62ced780d99f47b9b81a8792ebba0c81" + } + }, + "3f6f6916c88f4dc98d7352dedcf9904a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2be10f1a3776478e9c3abf135e19cf9d", + "placeholder": "​", + "style": "IPY_MODEL_b8f58385d9494a34be55835a1d05c1a9", + "value": "Downloading (…)l_00003-of-00010.bin: 100%" + } + }, + "7be9343698da48f09ad19226f8085b25": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6f2aa3118da34e5fb794d647d2080e00", + "max": 1308698073, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2275533744b944769822b83c60a8078b", + "value": 1308698073 + } + }, + "a1db2f79a1374fad9744771f92ecc939": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c5f4d79cad284d5db8364e02d09770d8", + "placeholder": "​", + "style": "IPY_MODEL_ad7d72ed5cf14dd6bc91284c1a256eab", + "value": " 1.31G/1.31G [00:36<00:00, 47.1MB/s]" + } + }, + "62ced780d99f47b9b81a8792ebba0c81": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2be10f1a3776478e9c3abf135e19cf9d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b8f58385d9494a34be55835a1d05c1a9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6f2aa3118da34e5fb794d647d2080e00": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2275533744b944769822b83c60a8078b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c5f4d79cad284d5db8364e02d09770d8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad7d72ed5cf14dd6bc91284c1a256eab": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "32ca4caaf0674d8cb52e40a29a86fc27": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_bbfbef8b919e42889fba89d5127bf530", + "IPY_MODEL_ea66a6c56417419385d4fad1be71f4ac", + "IPY_MODEL_0104669680f640adbd2be2756239074d" + ], + "layout": "IPY_MODEL_9667f6d64c4e4547965b07e524912eba" + } + }, + "bbfbef8b919e42889fba89d5127bf530": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c25b2f4f12864d11badcbd49769ce45d", + "placeholder": "​", + "style": "IPY_MODEL_1a5ea81653d046b6a62c86316ec745c0", + "value": "Downloading (…)l_00004-of-00010.bin: 100%" + } + }, + "ea66a6c56417419385d4fad1be71f4ac": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_07b530e101d543c18ccd8f7e629d2af7", + "max": 1832969625, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9ae74500398849f4bf51d1c2d805116e", + "value": 1832969625 + } + }, + "0104669680f640adbd2be2756239074d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_782182106c4c47588d1f2b2b4254f3af", + "placeholder": "​", + "style": "IPY_MODEL_d1a61f6f430b40ffa51729f17293ff6a", + "value": " 1.83G/1.83G [00:44<00:00, 43.0MB/s]" + } + }, + "9667f6d64c4e4547965b07e524912eba": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c25b2f4f12864d11badcbd49769ce45d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1a5ea81653d046b6a62c86316ec745c0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "07b530e101d543c18ccd8f7e629d2af7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9ae74500398849f4bf51d1c2d805116e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "782182106c4c47588d1f2b2b4254f3af": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d1a61f6f430b40ffa51729f17293ff6a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "aa912d9a8aed4ef2a20b94caa8991c88": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_94bbc4d0a7e741c38245deb28ba2d867", + "IPY_MODEL_9a138a4a9e0643a6a84dfe683e58b349", + "IPY_MODEL_88f664db527b4676ba596616dd5c76c9" + ], + "layout": "IPY_MODEL_85cb286a15ed40748a8bf9dc06bf25e0" + } + }, + "94bbc4d0a7e741c38245deb28ba2d867": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5dc4d92243fb47e9812f35b24d8cca70", + "placeholder": "​", + "style": "IPY_MODEL_07ae965b720f45c9822161410dec97ee", + "value": "Downloading (…)l_00005-of-00010.bin: 100%" + } + }, + "9a138a4a9e0643a6a84dfe683e58b349": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bc02d613994e41c69e838a42200cfbf0", + "max": 1350624729, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ad3c8520b2db4c09843e11420ad1514c", + "value": 1350624729 + } + }, + "88f664db527b4676ba596616dd5c76c9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_83a2c45423da4f0ea4d775ad41c03f51", + "placeholder": "​", + "style": "IPY_MODEL_20bd70096c434ea6b7de8441e09f00ef", + "value": " 1.35G/1.35G [00:30<00:00, 32.9MB/s]" + } + }, + "85cb286a15ed40748a8bf9dc06bf25e0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5dc4d92243fb47e9812f35b24d8cca70": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "07ae965b720f45c9822161410dec97ee": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc02d613994e41c69e838a42200cfbf0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad3c8520b2db4c09843e11420ad1514c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "83a2c45423da4f0ea4d775ad41c03f51": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "20bd70096c434ea6b7de8441e09f00ef": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bb55fef897484bd0a7119964d6e5fdd7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a8df56c1387646b59f617e89c89d1483", + "IPY_MODEL_820e0360dd9842c99eedcbdc04f70a16", + "IPY_MODEL_7c8c3f0ab1e5488fb843036409cd3cc6" + ], + "layout": "IPY_MODEL_6a8874c0e9584b2dbbbdc9b930e182b4" + } + }, + "a8df56c1387646b59f617e89c89d1483": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1bbaab9416f644dabbf64211ab3307c5", + "placeholder": "​", + "style": "IPY_MODEL_05c326736cda43ef9b0f9cc9da773a7c", + "value": "Downloading (…)l_00006-of-00010.bin: 100%" + } + }, + "820e0360dd9842c99eedcbdc04f70a16": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_476dd2fd197b40c980235b0d18dbc71a", + "max": 1350624729, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_fdf249a78291445b9e78458c806f02cc", + "value": 1350624729 + } + }, + "7c8c3f0ab1e5488fb843036409cd3cc6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c3db5cf8db8e498c82a1667738812d76", + "placeholder": "​", + "style": "IPY_MODEL_d51413d11ad94605add2c3fcec20f557", + "value": " 1.35G/1.35G [00:40<00:00, 25.1MB/s]" + } + }, + "6a8874c0e9584b2dbbbdc9b930e182b4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1bbaab9416f644dabbf64211ab3307c5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "05c326736cda43ef9b0f9cc9da773a7c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "476dd2fd197b40c980235b0d18dbc71a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fdf249a78291445b9e78458c806f02cc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c3db5cf8db8e498c82a1667738812d76": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d51413d11ad94605add2c3fcec20f557": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "62495c325f8449a0ae625442484d1b40": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_53866a430c6240319e78f039f750c715", + "IPY_MODEL_216f17794d534351b1e98bc0348cca3b", + "IPY_MODEL_7810c293152848429b119ab2949ac8e4" + ], + "layout": "IPY_MODEL_148772f9df134c62860b8926467a3953" + } + }, + "53866a430c6240319e78f039f750c715": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6f1cf12b706b46a085ea5003a9937337", + "placeholder": "​", + "style": "IPY_MODEL_2f8212af7e1d40c2aff385c9a1cbcd48", + "value": "Downloading (…)l_00007-of-00010.bin: 100%" + } + }, + "216f17794d534351b1e98bc0348cca3b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bcf6f0ca21a1451ba54850f9cb77fdc3", + "max": 1543562713, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9253baad46bb431e87908a246572f188", + "value": 1543562713 + } + }, + "7810c293152848429b119ab2949ac8e4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_01339fd6b29b49bf84ea3327946dff6d", + "placeholder": "​", + "style": "IPY_MODEL_e6dfd6adcf0f4b95ba774694c5bce62a", + "value": " 1.54G/1.54G [00:39<00:00, 35.5MB/s]" + } + }, + "148772f9df134c62860b8926467a3953": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6f1cf12b706b46a085ea5003a9937337": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2f8212af7e1d40c2aff385c9a1cbcd48": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bcf6f0ca21a1451ba54850f9cb77fdc3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9253baad46bb431e87908a246572f188": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "01339fd6b29b49bf84ea3327946dff6d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e6dfd6adcf0f4b95ba774694c5bce62a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4c54b563ff444bbbb722488f4f51aaaf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e8c5e684338b4d439c6308ed3f0d35b1", + "IPY_MODEL_f3f919800dd1410ebf47a7fc14e58eb2", + "IPY_MODEL_574c832a50214622b0660650cf58aedc" + ], + "layout": "IPY_MODEL_6d5c7155e8c048b89e997446ee54fa03" + } + }, + "e8c5e684338b4d439c6308ed3f0d35b1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_df164e2749b247418d58ad2a0654e8c9", + "placeholder": "​", + "style": "IPY_MODEL_61d2f13e80344bb9b2dc576825b080db", + "value": "Downloading (…)l_00008-of-00010.bin: 100%" + } + }, + "f3f919800dd1410ebf47a7fc14e58eb2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8b745ec7c29049da8789662d92fd2184", + "max": 1426130393, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_be4d0fb3c8b24892af37cf4700776e39", + "value": 1426130393 + } + }, + "574c832a50214622b0660650cf58aedc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_00a07262e0674357b8f95a6c1989dd2d", + "placeholder": "​", + "style": "IPY_MODEL_028b03f4d2cb4944b7202da0f374db5a", + "value": " 1.43G/1.43G [00:35<00:00, 47.0MB/s]" + } + }, + "6d5c7155e8c048b89e997446ee54fa03": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "df164e2749b247418d58ad2a0654e8c9": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "61d2f13e80344bb9b2dc576825b080db": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8b745ec7c29049da8789662d92fd2184": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "be4d0fb3c8b24892af37cf4700776e39": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "00a07262e0674357b8f95a6c1989dd2d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "028b03f4d2cb4944b7202da0f374db5a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6ebfcbce697c4d73ae82a13126f3bdbe": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_83c14f894c46488b965898032ae0051a", + "IPY_MODEL_5a5308f4fb444e759dd7127c77f76159", + "IPY_MODEL_8e9f05fd6d2b467181dd10927a8dc0b5" + ], + "layout": "IPY_MODEL_b920875c22144524a56922d61b4c799f" + } + }, + "83c14f894c46488b965898032ae0051a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6ab1aa45953c4ff599c1f04ef2788cda", + "placeholder": "​", + "style": "IPY_MODEL_69872c3ff0ef4dcbba323080f09e55c0", + "value": "Downloading (…)l_00009-of-00010.bin: 100%" + } + }, + "5a5308f4fb444e759dd7127c77f76159": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cdaafd60a6304da8b67731ff5fd69eb2", + "max": 1342244313, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4caa71e0a803473985bd3b776f86dba0", + "value": 1342244313 + } + }, + "8e9f05fd6d2b467181dd10927a8dc0b5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_add4ae4b461541f89d62bb964efcb1de", + "placeholder": "​", + "style": "IPY_MODEL_cfa1c1172d024e3e8d93aabba1b71dca", + "value": " 1.34G/1.34G [00:33<00:00, 35.3MB/s]" + } + }, + "b920875c22144524a56922d61b4c799f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6ab1aa45953c4ff599c1f04ef2788cda": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "69872c3ff0ef4dcbba323080f09e55c0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cdaafd60a6304da8b67731ff5fd69eb2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4caa71e0a803473985bd3b776f86dba0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "add4ae4b461541f89d62bb964efcb1de": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cfa1c1172d024e3e8d93aabba1b71dca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "62a852470dfd4142920ab9e2e4fd1dd8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a71702ae59c44a59bb487b978da7c6f7", + "IPY_MODEL_d907951756c34f81bf1d7f8309044edd", + "IPY_MODEL_2e55e4b579a447c8be995206ab04e0af" + ], + "layout": "IPY_MODEL_6217fad2a64a4c45889f79f955fadc94" + } + }, + "a71702ae59c44a59bb487b978da7c6f7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4961c4b533a34105b814702d2efc245f", + "placeholder": "​", + "style": "IPY_MODEL_f17f19e7cfed45a49c01dd0d2aa3c0ec", + "value": "Downloading (…)l_00010-of-00010.bin: 100%" + } + }, + "d907951756c34f81bf1d7f8309044edd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6c90d2d3c2fb450dbcb3cb34f9916d71", + "max": 1325458905, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3aa86ef05d5643448e0ffadb9ab906da", + "value": 1325458905 + } + }, + "2e55e4b579a447c8be995206ab04e0af": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c72551dd5314820b85241edf9a0eacf", + "placeholder": "​", + "style": "IPY_MODEL_9491de0faffb48ea813bc0087dd0382c", + "value": " 1.33G/1.33G [00:36<00:00, 48.5MB/s]" + } + }, + "6217fad2a64a4c45889f79f955fadc94": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4961c4b533a34105b814702d2efc245f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f17f19e7cfed45a49c01dd0d2aa3c0ec": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6c90d2d3c2fb450dbcb3cb34f9916d71": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3aa86ef05d5643448e0ffadb9ab906da": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0c72551dd5314820b85241edf9a0eacf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9491de0faffb48ea813bc0087dd0382c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b75e669da91b4b41a735714dedce5cbf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_88487e567f0d45ccb9df77da683b20fa", + "IPY_MODEL_857e51f071fc42a4beeb8696e0bf052a", + "IPY_MODEL_c14b47aab1974287aca22982fdea5d90" + ], + "layout": "IPY_MODEL_0b4cffa5a6944cf691d1d687f8e2606f" + } + }, + "88487e567f0d45ccb9df77da683b20fa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7bb9b1b90ca94d329d995e643b3d7900", + "placeholder": "​", + "style": "IPY_MODEL_1471ff412be64c6c9eaf6776234c7638", + "value": "Downloading (…)l_00011-of-00010.bin: 100%" + } + }, + "857e51f071fc42a4beeb8696e0bf052a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_60645ebcd09743fea162f76c29e90586", + "max": 33555370, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1673a7dcd19444039e03c77fb721a2b1", + "value": 33555370 + } + }, + "c14b47aab1974287aca22982fdea5d90": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_81bdf80fecc24f3b82675caa9e5a37bb", + "placeholder": "​", + "style": "IPY_MODEL_2c0b2843879644d0bce2e36ed67b070d", + "value": " 33.6M/33.6M [00:00<00:00, 38.4MB/s]" + } + }, + "0b4cffa5a6944cf691d1d687f8e2606f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7bb9b1b90ca94d329d995e643b3d7900": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1471ff412be64c6c9eaf6776234c7638": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "60645ebcd09743fea162f76c29e90586": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1673a7dcd19444039e03c77fb721a2b1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "81bdf80fecc24f3b82675caa9e5a37bb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2c0b2843879644d0bce2e36ed67b070d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ef4493b14f0c4cc0ac4c2ed064d0cd48": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2d7f293e1c164f349202726c00da6712", + "IPY_MODEL_02737142f97f4d1c92b56b13d3336cf9", + "IPY_MODEL_cd81996a8ca14147a0cd47be54d4c48b" + ], + "layout": "IPY_MODEL_c5aabd9e38a24cccb5fec66ade683da0" + } + }, + "2d7f293e1c164f349202726c00da6712": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6df33d7017fe45eaba1f2fe53b950e86", + "placeholder": "​", + "style": "IPY_MODEL_78a44cb210c24667a0af9a36e8f874dc", + "value": "Loading checkpoint shards: 100%" + } + }, + "02737142f97f4d1c92b56b13d3336cf9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2ea85bdaddb14497984d682b474b319f", + "max": 11, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b4f38921920f4179885a934dc84cb931", + "value": 11 + } + }, + "cd81996a8ca14147a0cd47be54d4c48b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a35a5003fb1542dab4daf80655e2c27e", + "placeholder": "​", + "style": "IPY_MODEL_0642fddfb2e745eabba8dc9c901166fe", + "value": " 11/11 [01:25<00:00, 5.35s/it]" + } + }, + "c5aabd9e38a24cccb5fec66ade683da0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6df33d7017fe45eaba1f2fe53b950e86": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "78a44cb210c24667a0af9a36e8f874dc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2ea85bdaddb14497984d682b474b319f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b4f38921920f4179885a934dc84cb931": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a35a5003fb1542dab4daf80655e2c27e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0642fddfb2e745eabba8dc9c901166fe": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "190071cc3deb4d5d8049e564c4f2dd4e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9ad5c71996bc4b69b2e2f6310004da03", + "IPY_MODEL_a13e14d450c842a09118ca9bbd093d46", + "IPY_MODEL_aafa76dc5b54491fae17460daf3895ae" + ], + "layout": "IPY_MODEL_56767cd778a043efa8290db597ee6faf" + } + }, + "9ad5c71996bc4b69b2e2f6310004da03": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6d073d625fb44bc2b8f16735c9e4702e", + "placeholder": "​", + "style": "IPY_MODEL_f4f88026f56444e6a6c7d2eeea78be07", + "value": "Downloading (…)neration_config.json: 100%" + } + }, + "a13e14d450c842a09118ca9bbd093d46": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e290f553855f459b95c1e0235f9cbf44", + "max": 116, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_71063624875440e5917e33b5c465041f", + "value": 116 + } + }, + "aafa76dc5b54491fae17460daf3895ae": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b4dcd2316ddf4c6e9d754200f73cfefb", + "placeholder": "​", + "style": "IPY_MODEL_41417176a77a48acb1216a8b95ddc0a7", + "value": " 116/116 [00:00<00:00, 8.67kB/s]" + } + }, + "56767cd778a043efa8290db597ee6faf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6d073d625fb44bc2b8f16735c9e4702e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4f88026f56444e6a6c7d2eeea78be07": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e290f553855f459b95c1e0235f9cbf44": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71063624875440e5917e33b5c465041f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b4dcd2316ddf4c6e9d754200f73cfefb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "41417176a77a48acb1216a8b95ddc0a7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git "a/examples/notebooks/\360\237\246\226_X\342\200\224LLM_Prototyping.ipynb" "b/examples/notebooks/\360\237\246\226_X\342\200\224LLM_Prototyping.ipynb" index 06bf130..8e216be 100644 --- "a/examples/notebooks/\360\237\246\226_X\342\200\224LLM_Prototyping.ipynb" +++ "b/examples/notebooks/\360\237\246\226_X\342\200\224LLM_Prototyping.ipynb" @@ -1,2730 +1,2730 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "gpuType": "T4", - "collapsed_sections": [ - "187Yhr0Hhs_r" - ] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4", + "collapsed_sections": [ + "187Yhr0Hhs_r" + ] }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# 🦖 X—LLM: Easy & Cutting Edge LLM Finetuning\n", - "\n", - "Tutorial how to run X—LLM in colab\n", - "\n", - "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", - "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", - "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", - "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", - " do\n", - "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", - " into your own project\n", - "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", - "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model\n" - ], - "metadata": { - "id": "nfE8HHxFECqI" - } - }, - { - "cell_type": "markdown", - "source": [ - "First of all you need to install the latest `xllm` version" - ], - "metadata": { - "id": "RK7mWDkLEqhX" - } - }, - { - "cell_type": "markdown", - "source": [ - "# Installation" - ], - "metadata": { - "id": "187Yhr0Hhs_r" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "7jluomCQ65wT", - "outputId": "89ec8415-9f01-422e-9597-bc46c2160559" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Requirement already satisfied: xllm in /usr/local/lib/python3.10/dist-packages (0.0.10)\n", - "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.23.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (23.2)\n", - "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from xllm) (5.9.5)\n", - "Requirement already satisfied: torch>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (2.1.0+cu118)\n", - "Requirement already satisfied: loguru in /usr/local/lib/python3.10/dist-packages (from xllm) (0.7.2)\n", - "Requirement already satisfied: peft>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (0.6.2)\n", - "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from xllm) (0.16.0)\n", - "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from xllm) (1.0.0)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from xllm) (2.31.0)\n", - "Requirement already satisfied: optimum>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.14.0)\n", - "Requirement already satisfied: bitsandbytes>=0.41.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (0.41.2.post2)\n", - "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from xllm) (1.11.3)\n", - "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from xllm) (4.35.1)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from xllm) (4.66.1)\n", - "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from xllm) (0.4.0)\n", - "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (15.0.1)\n", - "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (1.12)\n", - "Requirement already satisfied: huggingface-hub>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (0.17.3)\n", - "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (2.14.6)\n", - "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (6.0.1)\n", - "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (0.24.1)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.13.1)\n", - "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (4.5.0)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.1.2)\n", - "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2023.6.0)\n", - "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2.1.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2023.7.22)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (2023.6.3)\n", - "Requirement already satisfied: tokenizers<0.15,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (0.14.1)\n", - "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (8.1.7)\n", - "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.1.40)\n", - "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.35.0)\n", - "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (0.4.0)\n", - "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.3.3)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (67.7.2)\n", - "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.4.4)\n", - "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.20.3)\n", - "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb->xllm) (1.16.0)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->xllm) (4.0.11)\n", - "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (0.1.99)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum>=1.12.0->xllm) (10.0)\n", - "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (9.0.0)\n", - "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (0.3.7)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (1.5.3)\n", - "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.4.1)\n", - "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (0.70.15)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.8.6)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.1->xllm) (2.1.3)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum>=1.12.0->xllm) (1.3.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.3.1)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->xllm) (5.0.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2023.3.post1)\n" - ] - } - ], - "source": [ - "# default version\n", - "!pip install xllm\n", - "\n", - "# version which include deepspeed, flash-attn and auto-gptq\n", - "# !pip install xllm[train]" - ] + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# 🦖 X—LLM: Easy & Cutting Edge LLM Finetuning\n", + "\n", + "Tutorial how to run X—LLM in colab\n", + "\n", + "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", + "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", + "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", + "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", + " do\n", + "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", + " into your own project\n", + "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", + "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model\n" + ], + "metadata": { + "id": "nfE8HHxFECqI" + } + }, + { + "cell_type": "markdown", + "source": [ + "First of all you need to install the latest `xllm` version" + ], + "metadata": { + "id": "RK7mWDkLEqhX" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Installation" + ], + "metadata": { + "id": "187Yhr0Hhs_r" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "7jluomCQ65wT", + "outputId": "89ec8415-9f01-422e-9597-bc46c2160559" + }, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "# Verify the versions and confirm whether CUDA is available" - ], - "metadata": { - "id": "dOmUEQGPFSPO" - } + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: xllm in /usr/local/lib/python3.10/dist-packages (0.0.10)\n", + "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (23.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from xllm) (5.9.5)\n", + "Requirement already satisfied: torch>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (2.1.0+cu118)\n", + "Requirement already satisfied: loguru in /usr/local/lib/python3.10/dist-packages (from xllm) (0.7.2)\n", + "Requirement already satisfied: peft>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (0.6.2)\n", + "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from xllm) (0.16.0)\n", + "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from xllm) (1.0.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from xllm) (2.31.0)\n", + "Requirement already satisfied: optimum>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from xllm) (1.14.0)\n", + "Requirement already satisfied: bitsandbytes>=0.41.1 in /usr/local/lib/python3.10/dist-packages (from xllm) (0.41.2.post2)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from xllm) (1.11.3)\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from xllm) (4.35.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from xllm) (4.66.1)\n", + "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from xllm) (0.4.0)\n", + "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (15.0.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (1.12)\n", + "Requirement already satisfied: huggingface-hub>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (0.17.3)\n", + "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from optimum>=1.12.0->xllm) (2.14.6)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (6.0.1)\n", + "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from peft>=0.5.0->xllm) (0.24.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (4.5.0)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (3.1.2)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2023.6.0)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->xllm) (2.1.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->xllm) (2023.7.22)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (2023.6.3)\n", + "Requirement already satisfied: tokenizers<0.15,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (0.14.1)\n", + "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (8.1.7)\n", + "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.1.40)\n", + "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.35.0)\n", + "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (0.4.0)\n", + "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.3.3)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (67.7.2)\n", + "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (1.4.4)\n", + "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->xllm) (3.20.3)\n", + "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb->xllm) (1.16.0)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->xllm) (4.0.11)\n", + "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers->xllm) (0.1.99)\n", + "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum>=1.12.0->xllm) (10.0)\n", + "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (9.0.0)\n", + "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (0.3.7)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (1.5.3)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.4.1)\n", + "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (0.70.15)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum>=1.12.0->xllm) (3.8.6)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.1->xllm) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum>=1.12.0->xllm) (1.3.0)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (23.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum>=1.12.0->xllm) (1.3.1)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->xllm) (5.0.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum>=1.12.0->xllm) (2023.3.post1)\n" + ] + } + ], + "source": [ + "# default version\n", + "!pip install xllm\n", + "\n", + "# version which include deepspeed, flash-attn and auto-gptq\n", + "# !pip install xllm[train]" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Verify the versions and confirm whether CUDA is available" + ], + "metadata": { + "id": "dOmUEQGPFSPO" + } + }, + { + "cell_type": "code", + "source": [ + "import torch\n", + "import xllm\n", + "\n", + "cuda_is_available = torch.cuda.is_available()\n", + "\n", + "print(f\"X—LLM version: {xllm.__version__}\\nTorch version: {torch.__version__}\\nCuda is available: {cuda_is_available}\")\n", + "assert cuda_is_available" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "YpbY13qq7AIr", + "outputId": "d70b38c7-5f92-4a40-854e-4f4ecd92879a" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "code", - "source": [ - "import torch\n", - "import xllm\n", - "\n", - "cuda_is_available = torch.cuda.is_available()\n", - "\n", - "print(f\"X—LLM version: {xllm.__version__}\\nTorch version: {torch.__version__}\\nCuda is available: {cuda_is_available}\")\n", - "assert cuda_is_available" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "YpbY13qq7AIr", - "outputId": "d70b38c7-5f92-4a40-854e-4f4ecd92879a" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "X—LLM version: 0.0.10\n", - "Torch version: 2.1.0+cu118\n", - "Cuda is available: True\n" - ] - } - ] + "output_type": "stream", + "name": "stdout", + "text": [ + "X—LLM version: 0.0.10\n", + "Torch version: 2.1.0+cu118\n", + "Cuda is available: True\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Single cell example" + ], + "metadata": { + "id": "yX6IKojXS4JH" + } + }, + { + "cell_type": "code", + "source": [ + "from xllm import Config\n", + "from xllm.datasets import GeneralDataset\n", + "from xllm.experiments import Experiment\n", + "\n", + "# 1. Init Config which controls the internal logic of xllm\n", + "config = Config(\n", + " model_name_or_path=\"facebook/opt-350m\",\n", + " force_fp32=True, # only for colab\n", + ")\n", + "\n", + "# 2. Prepare the data\n", + "train_data = [\"Hello!\"] * 100\n", + "\n", + "# 3. Load the data\n", + "train_dataset = GeneralDataset.from_list(data=train_data)\n", + "\n", + "# 4. Init Experiment\n", + "experiment = Experiment(config=config, train_dataset=train_dataset)\n", + "\n", + "# 5. Build Experiment from Config: init tokenizer and model, apply LoRA and so on\n", + "experiment.build()\n", + "\n", + "# 6. Run Experiment (training)\n", + "experiment.run()\n", + "\n", + "# 7. [Optional] Fuse LoRA layers\n", + "# experiment.fuse_lora()\n", + "\n", + "# 8. [Optional] Push fused model (or just LoRA weight) to the HuggingFace Hub\n", + "# experiment.push_to_hub(repo_id=\"YOUR_NAME/MODEL_NAME\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 }, + "id": "hS32uIgmdOor", + "outputId": "5fca53b3-371f-4708-c3be-b9a9244cc2a0" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "# Single cell example" - ], - "metadata": { - "id": "yX6IKojXS4JH" - } + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 15:58:32.074\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.080\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", + "{\n", + " \"experiment_key\": \"base\",\n", + " \"save_safetensors\": true,\n", + " \"max_shard_size\": \"10GB\",\n", + " \"local_rank\": 0,\n", + " \"use_gradient_checkpointing\": false,\n", + " \"trainer_key\": \"lm\",\n", + " \"force_fp32\": true,\n", + " \"force_fp16\": false,\n", + " \"from_gptq\": false,\n", + " \"huggingface_hub_token\": null,\n", + " \"deepspeed_stage\": 0,\n", + " \"deepspeed_config_path\": null,\n", + " \"fsdp_strategy\": \"\",\n", + " \"fsdp_offload\": true,\n", + " \"seed\": 42,\n", + " \"stabilize\": false,\n", + " \"path_to_env_file\": \"./.env\",\n", + " \"prepare_dataset\": true,\n", + " \"lora_hub_model_id\": null,\n", + " \"lora_model_local_path\": null,\n", + " \"fused_model_local_path\": null,\n", + " \"fuse_after_training\": false,\n", + " \"quantization_dataset_id\": null,\n", + " \"quantization_max_samples\": 1024,\n", + " \"quantized_model_path\": \"./quantized_model/\",\n", + " \"quantized_hub_model_id\": null,\n", + " \"quantized_hub_private_repo\": true,\n", + " \"dataset_key\": \"soda\",\n", + " \"train_local_path_to_data\": \"./train.jsonl\",\n", + " \"eval_local_path_to_data\": null,\n", + " \"shuffle\": true,\n", + " \"max_eval_samples\": 1000,\n", + " \"add_eval_to_train_if_no_path\": false,\n", + " \"tokenizer_name_or_path\": null,\n", + " \"tokenizer_use_fast\": null,\n", + " \"tokenizer_padding_side\": null,\n", + " \"collator_key\": \"lm\",\n", + " \"max_length\": 2048,\n", + " \"model_name_or_path\": \"facebook/opt-350m\",\n", + " \"push_to_hub_bos_add_bos_token\": false,\n", + " \"use_flash_attention_2\": false,\n", + " \"trust_remote_code\": false,\n", + " \"device_map\": null,\n", + " \"prepare_model_for_kbit_training\": true,\n", + " \"load_in_8bit\": false,\n", + " \"load_in_4bit\": false,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_quantize_after_model_init\": false,\n", + " \"gptq_bits\": 4,\n", + " \"gptq_group_size\": 128,\n", + " \"gptq_disable_exllama\": true,\n", + " \"apply_lora\": false,\n", + " \"lora_rank\": 8,\n", + " \"lora_alpha\": 32,\n", + " \"lora_dropout\": 0.1,\n", + " \"raw_lora_target_modules\": \"all\",\n", + " \"output_dir\": \"./outputs/\",\n", + " \"per_device_train_batch_size\": 2,\n", + " \"do_eval\": false,\n", + " \"per_device_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": null,\n", + " \"eval_delay\": 0,\n", + " \"eval_steps\": 1000,\n", + " \"warmup_steps\": 1000,\n", + " \"max_steps\": null,\n", + " \"num_train_epochs\": 1,\n", + " \"learning_rate\": 0.0002,\n", + " \"max_grad_norm\": 1.0,\n", + " \"weight_decay\": 0.001,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"logging_steps\": 10,\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"push_to_hub\": false,\n", + " \"hub_model_id\": null,\n", + " \"hub_private_repo\": true,\n", + " \"report_to_wandb\": false,\n", + " \"wandb_api_key\": null,\n", + " \"wandb_project\": null,\n", + " \"wandb_entity\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.083\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.095\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", + "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", + "\u001b[32m2023-11-14 15:58:32.114\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", + "{\n", + " \"output_dir\": \"./outputs/\",\n", + " \"overwrite_output_dir\": false,\n", + " \"do_train\": false,\n", + " \"do_eval\": false,\n", + " \"do_predict\": false,\n", + " \"evaluation_strategy\": \"no\",\n", + " \"prediction_loss_only\": false,\n", + " \"per_device_train_batch_size\": 2,\n", + " \"per_device_eval_batch_size\": 2,\n", + " \"per_gpu_train_batch_size\": null,\n", + " \"per_gpu_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": 1,\n", + " \"eval_delay\": 0,\n", + " \"learning_rate\": 0.0002,\n", + " \"weight_decay\": 0.001,\n", + " \"adam_beta1\": 0.9,\n", + " \"adam_beta2\": 0.999,\n", + " \"adam_epsilon\": 1e-08,\n", + " \"max_grad_norm\": 1.0,\n", + " \"num_train_epochs\": 1,\n", + " \"max_steps\": -1,\n", + " \"lr_scheduler_type\": \"linear\",\n", + " \"warmup_ratio\": 0.0,\n", + " \"warmup_steps\": 1000,\n", + " \"log_level\": \"info\",\n", + " \"log_level_replica\": \"warning\",\n", + " \"log_on_each_node\": true,\n", + " \"logging_dir\": \"./outputs/runs/Nov14_15-58-32_735f762378cc\",\n", + " \"logging_strategy\": \"steps\",\n", + " \"logging_first_step\": true,\n", + " \"logging_steps\": 10,\n", + " \"logging_nan_inf_filter\": true,\n", + " \"save_strategy\": \"steps\",\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"save_safetensors\": true,\n", + " \"save_on_each_node\": false,\n", + " \"no_cuda\": false,\n", + " \"use_cpu\": false,\n", + " \"use_mps_device\": false,\n", + " \"seed\": 42,\n", + " \"data_seed\": 42,\n", + " \"jit_mode_eval\": false,\n", + " \"use_ipex\": false,\n", + " \"bf16\": false,\n", + " \"fp16\": true,\n", + " \"fp16_opt_level\": \"O1\",\n", + " \"half_precision_backend\": \"auto\",\n", + " \"bf16_full_eval\": false,\n", + " \"fp16_full_eval\": false,\n", + " \"tf32\": null,\n", + " \"local_rank\": 0,\n", + " \"ddp_backend\": null,\n", + " \"tpu_num_cores\": null,\n", + " \"tpu_metrics_debug\": false,\n", + " \"debug\": [],\n", + " \"dataloader_drop_last\": false,\n", + " \"eval_steps\": 1000,\n", + " \"dataloader_num_workers\": 0,\n", + " \"past_index\": -1,\n", + " \"run_name\": \"./outputs/\",\n", + " \"disable_tqdm\": false,\n", + " \"remove_unused_columns\": false,\n", + " \"label_names\": null,\n", + " \"load_best_model_at_end\": false,\n", + " \"metric_for_best_model\": \"loss\",\n", + " \"greater_is_better\": false,\n", + " \"ignore_data_skip\": false,\n", + " \"fsdp\": [],\n", + " \"fsdp_min_num_params\": 0,\n", + " \"fsdp_config\": {\n", + " \"min_num_params\": 0,\n", + " \"xla\": false,\n", + " \"xla_fsdp_grad_ckpt\": false\n", + " },\n", + " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", + " \"deepspeed\": null,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"optim_args\": null,\n", + " \"adafactor\": false,\n", + " \"group_by_length\": false,\n", + " \"length_column_name\": \"length\",\n", + " \"report_to\": [\n", + " \"tensorboard\"\n", + " ],\n", + " \"ddp_find_unused_parameters\": null,\n", + " \"ddp_bucket_cap_mb\": null,\n", + " \"ddp_broadcast_buffers\": null,\n", + " \"dataloader_pin_memory\": true,\n", + " \"skip_memory_metrics\": true,\n", + " \"use_legacy_prediction_loop\": false,\n", + " \"push_to_hub\": false,\n", + " \"resume_from_checkpoint\": null,\n", + " \"hub_model_id\": null,\n", + " \"hub_strategy\": \"checkpoint\",\n", + " \"hub_token\": \"\",\n", + " \"hub_private_repo\": true,\n", + " \"hub_always_push\": false,\n", + " \"gradient_checkpointing\": false,\n", + " \"gradient_checkpointing_kwargs\": null,\n", + " \"include_inputs_for_metrics\": false,\n", + " \"fp16_backend\": \"auto\",\n", + " \"push_to_hub_model_id\": null,\n", + " \"push_to_hub_organization\": null,\n", + " \"push_to_hub_token\": \"\",\n", + " \"mp_parameters\": \"\",\n", + " \"auto_find_batch_size\": false,\n", + " \"full_determinism\": false,\n", + " \"torchdynamo\": null,\n", + " \"ray_scope\": \"last\",\n", + " \"ddp_timeout\": 1800,\n", + " \"torch_compile\": false,\n", + " \"torch_compile_backend\": null,\n", + " \"torch_compile_mode\": null,\n", + " \"dispatch_batches\": null,\n", + " \"split_batches\": false,\n", + " \"include_tokens_per_second\": false,\n", + " \"neftune_noise_alpha\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.116\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.762\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.769\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:32.774\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config is None. Model will be loaded using torch.float32\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:40.542\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", + "Using auto half precision backend\n", + "\u001b[32m2023-11-14 15:58:45.117\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:45.120\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n", + "\u001b[32m2023-11-14 15:58:45.122\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", + "***** Running training *****\n", + " Num examples = 100\n", + " Num Epochs = 1\n", + " Instantaneous batch size per device = 2\n", + " Total train batch size (w. parallel, distributed & accumulation) = 2\n", + " Gradient Accumulation steps = 1\n", + " Total optimization steps = 50\n", + " Number of trainable parameters = 331,196,416\n" + ] }, { - "cell_type": "code", - "source": [ - "from xllm import Config\n", - "from xllm.datasets import GeneralDataset\n", - "from xllm.experiments import Experiment\n", - "\n", - "# 1. Init Config which controls the internal logic of xllm\n", - "config = Config(\n", - " model_name_or_path=\"facebook/opt-350m\",\n", - " force_fp32=True, # only for colab\n", - ")\n", - "\n", - "# 2. Prepare the data\n", - "train_data = [\"Hello!\"] * 100\n", - "\n", - "# 3. Load the data\n", - "train_dataset = GeneralDataset.from_list(data=train_data)\n", - "\n", - "# 4. Init Experiment\n", - "experiment = Experiment(config=config, train_dataset=train_dataset)\n", - "\n", - "# 5. Build Experiment from Config: init tokenizer and model, apply LoRA and so on\n", - "experiment.build()\n", - "\n", - "# 6. Run Experiment (training)\n", - "experiment.run()\n", - "\n", - "# 7. [Optional] Fuse LoRA layers\n", - "# experiment.fuse_lora()\n", - "\n", - "# 8. [Optional] Push fused model (or just LoRA weight) to the HuggingFace Hub\n", - "# experiment.push_to_hub(repo_id=\"YOUR_NAME/MODEL_NAME\")" + "output_type": "display_data", + "data": { + "text/plain": [ + "" ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "hS32uIgmdOor", - "outputId": "5fca53b3-371f-4708-c3be-b9a9244cc2a0" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 15:58:32.074\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.080\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", - "{\n", - " \"experiment_key\": \"base\",\n", - " \"save_safetensors\": true,\n", - " \"max_shard_size\": \"10GB\",\n", - " \"local_rank\": 0,\n", - " \"use_gradient_checkpointing\": false,\n", - " \"trainer_key\": \"lm\",\n", - " \"force_fp32\": true,\n", - " \"force_fp16\": false,\n", - " \"from_gptq\": false,\n", - " \"huggingface_hub_token\": null,\n", - " \"deepspeed_stage\": 0,\n", - " \"deepspeed_config_path\": null,\n", - " \"fsdp_strategy\": \"\",\n", - " \"fsdp_offload\": true,\n", - " \"seed\": 42,\n", - " \"stabilize\": false,\n", - " \"path_to_env_file\": \"./.env\",\n", - " \"prepare_dataset\": true,\n", - " \"lora_hub_model_id\": null,\n", - " \"lora_model_local_path\": null,\n", - " \"fused_model_local_path\": null,\n", - " \"fuse_after_training\": false,\n", - " \"quantization_dataset_id\": null,\n", - " \"quantization_max_samples\": 1024,\n", - " \"quantized_model_path\": \"./quantized_model/\",\n", - " \"quantized_hub_model_id\": null,\n", - " \"quantized_hub_private_repo\": true,\n", - " \"dataset_key\": \"soda\",\n", - " \"train_local_path_to_data\": \"./train.jsonl\",\n", - " \"eval_local_path_to_data\": null,\n", - " \"shuffle\": true,\n", - " \"max_eval_samples\": 1000,\n", - " \"add_eval_to_train_if_no_path\": false,\n", - " \"tokenizer_name_or_path\": null,\n", - " \"tokenizer_use_fast\": null,\n", - " \"tokenizer_padding_side\": null,\n", - " \"collator_key\": \"lm\",\n", - " \"max_length\": 2048,\n", - " \"model_name_or_path\": \"facebook/opt-350m\",\n", - " \"push_to_hub_bos_add_bos_token\": false,\n", - " \"use_flash_attention_2\": false,\n", - " \"trust_remote_code\": false,\n", - " \"device_map\": null,\n", - " \"prepare_model_for_kbit_training\": true,\n", - " \"load_in_8bit\": false,\n", - " \"load_in_4bit\": false,\n", - " \"llm_int8_threshold\": 6.0,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_quantize_after_model_init\": false,\n", - " \"gptq_bits\": 4,\n", - " \"gptq_group_size\": 128,\n", - " \"gptq_disable_exllama\": true,\n", - " \"apply_lora\": false,\n", - " \"lora_rank\": 8,\n", - " \"lora_alpha\": 32,\n", - " \"lora_dropout\": 0.1,\n", - " \"raw_lora_target_modules\": \"all\",\n", - " \"output_dir\": \"./outputs/\",\n", - " \"per_device_train_batch_size\": 2,\n", - " \"do_eval\": false,\n", - " \"per_device_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": null,\n", - " \"eval_delay\": 0,\n", - " \"eval_steps\": 1000,\n", - " \"warmup_steps\": 1000,\n", - " \"max_steps\": null,\n", - " \"num_train_epochs\": 1,\n", - " \"learning_rate\": 0.0002,\n", - " \"max_grad_norm\": 1.0,\n", - " \"weight_decay\": 0.001,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"logging_steps\": 10,\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"push_to_hub\": false,\n", - " \"hub_model_id\": null,\n", - " \"hub_private_repo\": true,\n", - " \"report_to_wandb\": false,\n", - " \"wandb_api_key\": null,\n", - " \"wandb_project\": null,\n", - " \"wandb_entity\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.083\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.095\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", - "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", - "\u001b[32m2023-11-14 15:58:32.114\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", - "{\n", - " \"output_dir\": \"./outputs/\",\n", - " \"overwrite_output_dir\": false,\n", - " \"do_train\": false,\n", - " \"do_eval\": false,\n", - " \"do_predict\": false,\n", - " \"evaluation_strategy\": \"no\",\n", - " \"prediction_loss_only\": false,\n", - " \"per_device_train_batch_size\": 2,\n", - " \"per_device_eval_batch_size\": 2,\n", - " \"per_gpu_train_batch_size\": null,\n", - " \"per_gpu_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": 1,\n", - " \"eval_delay\": 0,\n", - " \"learning_rate\": 0.0002,\n", - " \"weight_decay\": 0.001,\n", - " \"adam_beta1\": 0.9,\n", - " \"adam_beta2\": 0.999,\n", - " \"adam_epsilon\": 1e-08,\n", - " \"max_grad_norm\": 1.0,\n", - " \"num_train_epochs\": 1,\n", - " \"max_steps\": -1,\n", - " \"lr_scheduler_type\": \"linear\",\n", - " \"warmup_ratio\": 0.0,\n", - " \"warmup_steps\": 1000,\n", - " \"log_level\": \"info\",\n", - " \"log_level_replica\": \"warning\",\n", - " \"log_on_each_node\": true,\n", - " \"logging_dir\": \"./outputs/runs/Nov14_15-58-32_735f762378cc\",\n", - " \"logging_strategy\": \"steps\",\n", - " \"logging_first_step\": true,\n", - " \"logging_steps\": 10,\n", - " \"logging_nan_inf_filter\": true,\n", - " \"save_strategy\": \"steps\",\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"save_safetensors\": true,\n", - " \"save_on_each_node\": false,\n", - " \"no_cuda\": false,\n", - " \"use_cpu\": false,\n", - " \"use_mps_device\": false,\n", - " \"seed\": 42,\n", - " \"data_seed\": 42,\n", - " \"jit_mode_eval\": false,\n", - " \"use_ipex\": false,\n", - " \"bf16\": false,\n", - " \"fp16\": true,\n", - " \"fp16_opt_level\": \"O1\",\n", - " \"half_precision_backend\": \"auto\",\n", - " \"bf16_full_eval\": false,\n", - " \"fp16_full_eval\": false,\n", - " \"tf32\": null,\n", - " \"local_rank\": 0,\n", - " \"ddp_backend\": null,\n", - " \"tpu_num_cores\": null,\n", - " \"tpu_metrics_debug\": false,\n", - " \"debug\": [],\n", - " \"dataloader_drop_last\": false,\n", - " \"eval_steps\": 1000,\n", - " \"dataloader_num_workers\": 0,\n", - " \"past_index\": -1,\n", - " \"run_name\": \"./outputs/\",\n", - " \"disable_tqdm\": false,\n", - " \"remove_unused_columns\": false,\n", - " \"label_names\": null,\n", - " \"load_best_model_at_end\": false,\n", - " \"metric_for_best_model\": \"loss\",\n", - " \"greater_is_better\": false,\n", - " \"ignore_data_skip\": false,\n", - " \"fsdp\": [],\n", - " \"fsdp_min_num_params\": 0,\n", - " \"fsdp_config\": {\n", - " \"min_num_params\": 0,\n", - " \"xla\": false,\n", - " \"xla_fsdp_grad_ckpt\": false\n", - " },\n", - " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", - " \"deepspeed\": null,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"optim_args\": null,\n", - " \"adafactor\": false,\n", - " \"group_by_length\": false,\n", - " \"length_column_name\": \"length\",\n", - " \"report_to\": [\n", - " \"tensorboard\"\n", - " ],\n", - " \"ddp_find_unused_parameters\": null,\n", - " \"ddp_bucket_cap_mb\": null,\n", - " \"ddp_broadcast_buffers\": null,\n", - " \"dataloader_pin_memory\": true,\n", - " \"skip_memory_metrics\": true,\n", - " \"use_legacy_prediction_loop\": false,\n", - " \"push_to_hub\": false,\n", - " \"resume_from_checkpoint\": null,\n", - " \"hub_model_id\": null,\n", - " \"hub_strategy\": \"checkpoint\",\n", - " \"hub_token\": \"\",\n", - " \"hub_private_repo\": true,\n", - " \"hub_always_push\": false,\n", - " \"gradient_checkpointing\": false,\n", - " \"gradient_checkpointing_kwargs\": null,\n", - " \"include_inputs_for_metrics\": false,\n", - " \"fp16_backend\": \"auto\",\n", - " \"push_to_hub_model_id\": null,\n", - " \"push_to_hub_organization\": null,\n", - " \"push_to_hub_token\": \"\",\n", - " \"mp_parameters\": \"\",\n", - " \"auto_find_batch_size\": false,\n", - " \"full_determinism\": false,\n", - " \"torchdynamo\": null,\n", - " \"ray_scope\": \"last\",\n", - " \"ddp_timeout\": 1800,\n", - " \"torch_compile\": false,\n", - " \"torch_compile_backend\": null,\n", - " \"torch_compile_mode\": null,\n", - " \"dispatch_batches\": null,\n", - " \"split_batches\": false,\n", - " \"include_tokens_per_second\": false,\n", - " \"neftune_noise_alpha\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.116\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.762\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.769\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:32.774\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config is None. Model will be loaded using torch.float32\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:40.542\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", - "Using auto half precision backend\n", - "\u001b[32m2023-11-14 15:58:45.117\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:45.120\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n", - "\u001b[32m2023-11-14 15:58:45.122\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", - "***** Running training *****\n", - " Num examples = 100\n", - " Num Epochs = 1\n", - " Instantaneous batch size per device = 2\n", - " Total train batch size (w. parallel, distributed & accumulation) = 2\n", - " Gradient Accumulation steps = 1\n", - " Total optimization steps = 50\n", - " Number of trainable parameters = 331,196,416\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "

\n", - " \n", - " \n", - " [50/50 00:19, Epoch 1/1]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining Loss
15.197000
104.931000
203.759800
300.693400
400.167400
500.001100

" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "\n", - "Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\n", - "\n", - "\u001b[32m2023-11-14 15:59:06.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:06.622\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" - ] - } + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [50/50 00:19, Epoch 1/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
15.197000
104.931000
203.759800
300.693400
400.167400
500.001100

" ] + }, + "metadata": {} }, { - "cell_type": "markdown", - "source": [ - "# Add LoRA" - ], - "metadata": { - "id": "Cr2WobXNrInd" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Config\n", - "\n", - "`Config` plays a crucial role in the `xllm` library. It's how we define the workings of the library components, like how to handle data, the methods for training, the type of model to train, and so forth." - ], - "metadata": { - "id": "PJHflQLiFznP" - } - }, - { - "cell_type": "code", - "source": [ - "# config with LoRA\n", - "config = Config(\n", - " model_name_or_path=\"facebook/opt-350m\",\n", - " stabilize=True,\n", - " apply_lora=True,\n", - ")" - ], - "metadata": { - "id": "5wRPWEC_7ACr" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### You can explicitly specify the values of additional parameters in LoRA" - ], - "metadata": { - "id": "qrovyDw2uMzs" - } - }, - { - "cell_type": "code", - "source": [ - "# # extended config with LoRA\n", - "# config = Config(\n", - "# model_name_or_path=\"facebook/opt-350m\",\n", - "# stabilize=True,\n", - "# apply_lora=True,\n", - "# lora_rank=8,\n", - "# lora_alpha=32,\n", - "# lora_dropout=0.05,\n", - "# raw_lora_target_modules=\"all\",\n", - "# )" - ], - "metadata": { - "id": "PDZSBDkPuLTz" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Make training data" - ], - "metadata": { - "id": "vlu2UHTbuLAM" - } - }, - { - "cell_type": "code", - "source": [ - "train_data = [\"Hello!\", \"How are you?\", \"Are you okay?\"] * 100" - ], - "metadata": { - "id": "1oUYinonrZP-" - }, - "execution_count": null, - "outputs": [] + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "\u001b[32m2023-11-14 15:59:06.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:06.622\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Add LoRA" + ], + "metadata": { + "id": "Cr2WobXNrInd" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Config\n", + "\n", + "`Config` plays a crucial role in the `xllm` library. It's how we define the workings of the library components, like how to handle data, the methods for training, the type of model to train, and so forth." + ], + "metadata": { + "id": "PJHflQLiFznP" + } + }, + { + "cell_type": "code", + "source": [ + "# config with LoRA\n", + "config = Config(\n", + " model_name_or_path=\"facebook/opt-350m\",\n", + " stabilize=True,\n", + " apply_lora=True,\n", + ")" + ], + "metadata": { + "id": "5wRPWEC_7ACr" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### You can explicitly specify the values of additional parameters in LoRA" + ], + "metadata": { + "id": "qrovyDw2uMzs" + } + }, + { + "cell_type": "code", + "source": [ + "# # extended config with LoRA\n", + "# config = Config(\n", + "# model_name_or_path=\"facebook/opt-350m\",\n", + "# stabilize=True,\n", + "# apply_lora=True,\n", + "# lora_rank=8,\n", + "# lora_alpha=32,\n", + "# lora_dropout=0.05,\n", + "# raw_lora_target_modules=\"all\",\n", + "# )" + ], + "metadata": { + "id": "PDZSBDkPuLTz" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Make training data" + ], + "metadata": { + "id": "vlu2UHTbuLAM" + } + }, + { + "cell_type": "code", + "source": [ + "train_data = [\"Hello!\", \"How are you?\", \"Are you okay?\"] * 100" + ], + "metadata": { + "id": "1oUYinonrZP-" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "len(train_data)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "dE8GhcaFHNq4", + "outputId": "b0a55d10-4db6-4e0a-a04a-38a1f778cdd6" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "code", - "source": [ - "len(train_data)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dE8GhcaFHNq4", - "outputId": "b0a55d10-4db6-4e0a-a04a-38a1f778cdd6" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "300" - ] - }, - "metadata": {}, - "execution_count": 7 - } + "output_type": "execute_result", + "data": { + "text/plain": [ + "300" ] + }, + "metadata": {}, + "execution_count": 7 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Make a `xllm` train dataset" + ], + "metadata": { + "id": "LHE8Wxh2IKbV" + } + }, + { + "cell_type": "code", + "source": [ + "train_dataset = GeneralDataset.from_list(data=train_data)" + ], + "metadata": { + "id": "goJVNKvW6_81" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Init the experiment\n", + "\n", + "`Experiment` encompasses all aspects of training, such as how to load the model, whether to use LoRA or not, and how to set up the trainer, among other things.\n", + "\n", + "Required field is `config`.\n", + "\n", + "You can also pass the arguments that are listed below. Default value for each component is `None`.\n", + "\n", + "If you do not explicitly specify the value when initializing the experiment (that is, by default it will be `None`), then `Experiment` in step `.build` initializes the necessary components by referring to `Config` such as `tokenizer`, `model`, and so on.\n", + "```\n", + "training_arguments: Optional[TrainingArguments]\n", + "train_dataset: Optional[BaseDataset]\n", + "eval_dataset: Optional[BaseDataset]\n", + "tokenizer: Optional[PreTrainedTokenizer]\n", + "collator: Optional[BaseCollator]\n", + "quantization_config: Union[BitsAndBytesConfig, GPTQConfig, None]\n", + "model: Union[PreTrainedModel, PeftModel, None]\n", + "lora_config: Optional[LoraConfig]\n", + "trainer: Optional[LMTrainer]\n", + "```" + ], + "metadata": { + "id": "FjOvBBY6Iylu" + } + }, + { + "cell_type": "code", + "source": [ + "experiment = Experiment(config=config, train_dataset=train_dataset)" + ], + "metadata": { + "id": "q051EACF6_6F" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## 🏗 Build the experiment\n", + "\n", + "At this point, we're setting up all the components needed for training." + ], + "metadata": { + "id": "c_LNu8E4JPIW" + } + }, + { + "cell_type": "code", + "source": [ + "experiment.build()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "A_n-_AZ56_2z", + "outputId": "2cf97d7a-9cdc-4d15-e3aa-c328fcf65a42" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "## Make a `xllm` train dataset" - ], - "metadata": { - "id": "LHE8Wxh2IKbV" - } - }, - { - "cell_type": "code", - "source": [ - "train_dataset = GeneralDataset.from_list(data=train_data)" - ], - "metadata": { - "id": "goJVNKvW6_81" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Init the experiment\n", - "\n", - "`Experiment` encompasses all aspects of training, such as how to load the model, whether to use LoRA or not, and how to set up the trainer, among other things.\n", - "\n", - "Required field is `config`.\n", - "\n", - "You can also pass the arguments that are listed below. Default value for each component is `None`.\n", - "\n", - "If you do not explicitly specify the value when initializing the experiment (that is, by default it will be `None`), then `Experiment` in step `.build` initializes the necessary components by referring to `Config` such as `tokenizer`, `model`, and so on.\n", - "```\n", - "training_arguments: Optional[TrainingArguments]\n", - "train_dataset: Optional[BaseDataset]\n", - "eval_dataset: Optional[BaseDataset]\n", - "tokenizer: Optional[PreTrainedTokenizer]\n", - "collator: Optional[BaseCollator]\n", - "quantization_config: Union[BitsAndBytesConfig, GPTQConfig, None]\n", - "model: Union[PreTrainedModel, PeftModel, None]\n", - "lora_config: Optional[LoraConfig]\n", - "trainer: Optional[LMTrainer]\n", - "```" - ], - "metadata": { - "id": "FjOvBBY6Iylu" - } - }, - { - "cell_type": "code", - "source": [ - "experiment = Experiment(config=config, train_dataset=train_dataset)" - ], - "metadata": { - "id": "q051EACF6_6F" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## 🏗 Build the experiment\n", - "\n", - "At this point, we're setting up all the components needed for training." - ], - "metadata": { - "id": "c_LNu8E4JPIW" - } - }, - { - "cell_type": "code", - "source": [ - "experiment.build()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "A_n-_AZ56_2z", - "outputId": "2cf97d7a-9cdc-4d15-e3aa-c328fcf65a42" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 15:59:06.695\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:06.699\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", - "{\n", - " \"experiment_key\": \"base\",\n", - " \"save_safetensors\": true,\n", - " \"max_shard_size\": \"10GB\",\n", - " \"local_rank\": 0,\n", - " \"use_gradient_checkpointing\": false,\n", - " \"trainer_key\": \"lm\",\n", - " \"force_fp32\": false,\n", - " \"force_fp16\": false,\n", - " \"from_gptq\": false,\n", - " \"huggingface_hub_token\": null,\n", - " \"deepspeed_stage\": 0,\n", - " \"deepspeed_config_path\": null,\n", - " \"fsdp_strategy\": \"\",\n", - " \"fsdp_offload\": true,\n", - " \"seed\": 42,\n", - " \"stabilize\": true,\n", - " \"path_to_env_file\": \"./.env\",\n", - " \"prepare_dataset\": true,\n", - " \"lora_hub_model_id\": null,\n", - " \"lora_model_local_path\": null,\n", - " \"fused_model_local_path\": null,\n", - " \"fuse_after_training\": false,\n", - " \"quantization_dataset_id\": null,\n", - " \"quantization_max_samples\": 1024,\n", - " \"quantized_model_path\": \"./quantized_model/\",\n", - " \"quantized_hub_model_id\": null,\n", - " \"quantized_hub_private_repo\": true,\n", - " \"dataset_key\": \"soda\",\n", - " \"train_local_path_to_data\": \"./train.jsonl\",\n", - " \"eval_local_path_to_data\": null,\n", - " \"shuffle\": true,\n", - " \"max_eval_samples\": 1000,\n", - " \"add_eval_to_train_if_no_path\": false,\n", - " \"tokenizer_name_or_path\": null,\n", - " \"tokenizer_use_fast\": null,\n", - " \"tokenizer_padding_side\": null,\n", - " \"collator_key\": \"lm\",\n", - " \"max_length\": 2048,\n", - " \"model_name_or_path\": \"facebook/opt-350m\",\n", - " \"push_to_hub_bos_add_bos_token\": false,\n", - " \"use_flash_attention_2\": false,\n", - " \"trust_remote_code\": false,\n", - " \"device_map\": null,\n", - " \"prepare_model_for_kbit_training\": true,\n", - " \"load_in_8bit\": false,\n", - " \"load_in_4bit\": false,\n", - " \"llm_int8_threshold\": 6.0,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_quantize_after_model_init\": false,\n", - " \"gptq_bits\": 4,\n", - " \"gptq_group_size\": 128,\n", - " \"gptq_disable_exllama\": true,\n", - " \"apply_lora\": true,\n", - " \"lora_rank\": 8,\n", - " \"lora_alpha\": 32,\n", - " \"lora_dropout\": 0.1,\n", - " \"raw_lora_target_modules\": \"all\",\n", - " \"output_dir\": \"./outputs/\",\n", - " \"per_device_train_batch_size\": 2,\n", - " \"do_eval\": false,\n", - " \"per_device_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": null,\n", - " \"eval_delay\": 0,\n", - " \"eval_steps\": 1000,\n", - " \"warmup_steps\": 1000,\n", - " \"max_steps\": null,\n", - " \"num_train_epochs\": 1,\n", - " \"learning_rate\": 0.0002,\n", - " \"max_grad_norm\": 1.0,\n", - " \"weight_decay\": 0.001,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"logging_steps\": 10,\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"push_to_hub\": false,\n", - " \"hub_model_id\": null,\n", - " \"hub_private_repo\": true,\n", - " \"report_to_wandb\": false,\n", - " \"wandb_api_key\": null,\n", - " \"wandb_project\": null,\n", - " \"wandb_entity\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:06.700\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:06.709\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", - "PyTorch: setting up devices\n", - "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", - "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", - "\u001b[32m2023-11-14 15:59:06.722\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", - "{\n", - " \"output_dir\": \"./outputs/\",\n", - " \"overwrite_output_dir\": false,\n", - " \"do_train\": false,\n", - " \"do_eval\": false,\n", - " \"do_predict\": false,\n", - " \"evaluation_strategy\": \"no\",\n", - " \"prediction_loss_only\": false,\n", - " \"per_device_train_batch_size\": 2,\n", - " \"per_device_eval_batch_size\": 2,\n", - " \"per_gpu_train_batch_size\": null,\n", - " \"per_gpu_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": 1,\n", - " \"eval_delay\": 0,\n", - " \"learning_rate\": 0.0002,\n", - " \"weight_decay\": 0.001,\n", - " \"adam_beta1\": 0.9,\n", - " \"adam_beta2\": 0.999,\n", - " \"adam_epsilon\": 1e-08,\n", - " \"max_grad_norm\": 1.0,\n", - " \"num_train_epochs\": 1,\n", - " \"max_steps\": -1,\n", - " \"lr_scheduler_type\": \"linear\",\n", - " \"warmup_ratio\": 0.0,\n", - " \"warmup_steps\": 1000,\n", - " \"log_level\": \"info\",\n", - " \"log_level_replica\": \"warning\",\n", - " \"log_on_each_node\": true,\n", - " \"logging_dir\": \"./outputs/runs/Nov14_15-59-06_735f762378cc\",\n", - " \"logging_strategy\": \"steps\",\n", - " \"logging_first_step\": true,\n", - " \"logging_steps\": 10,\n", - " \"logging_nan_inf_filter\": true,\n", - " \"save_strategy\": \"steps\",\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"save_safetensors\": true,\n", - " \"save_on_each_node\": false,\n", - " \"no_cuda\": false,\n", - " \"use_cpu\": false,\n", - " \"use_mps_device\": false,\n", - " \"seed\": 42,\n", - " \"data_seed\": 42,\n", - " \"jit_mode_eval\": false,\n", - " \"use_ipex\": false,\n", - " \"bf16\": false,\n", - " \"fp16\": true,\n", - " \"fp16_opt_level\": \"O1\",\n", - " \"half_precision_backend\": \"auto\",\n", - " \"bf16_full_eval\": false,\n", - " \"fp16_full_eval\": false,\n", - " \"tf32\": null,\n", - " \"local_rank\": 0,\n", - " \"ddp_backend\": null,\n", - " \"tpu_num_cores\": null,\n", - " \"tpu_metrics_debug\": false,\n", - " \"debug\": [],\n", - " \"dataloader_drop_last\": false,\n", - " \"eval_steps\": 1000,\n", - " \"dataloader_num_workers\": 0,\n", - " \"past_index\": -1,\n", - " \"run_name\": \"./outputs/\",\n", - " \"disable_tqdm\": false,\n", - " \"remove_unused_columns\": false,\n", - " \"label_names\": null,\n", - " \"load_best_model_at_end\": false,\n", - " \"metric_for_best_model\": \"loss\",\n", - " \"greater_is_better\": false,\n", - " \"ignore_data_skip\": false,\n", - " \"fsdp\": [],\n", - " \"fsdp_min_num_params\": 0,\n", - " \"fsdp_config\": {\n", - " \"min_num_params\": 0,\n", - " \"xla\": false,\n", - " \"xla_fsdp_grad_ckpt\": false\n", - " },\n", - " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", - " \"deepspeed\": null,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"optim_args\": null,\n", - " \"adafactor\": false,\n", - " \"group_by_length\": false,\n", - " \"length_column_name\": \"length\",\n", - " \"report_to\": [\n", - " \"tensorboard\"\n", - " ],\n", - " \"ddp_find_unused_parameters\": null,\n", - " \"ddp_bucket_cap_mb\": null,\n", - " \"ddp_broadcast_buffers\": null,\n", - " \"dataloader_pin_memory\": true,\n", - " \"skip_memory_metrics\": true,\n", - " \"use_legacy_prediction_loop\": false,\n", - " \"push_to_hub\": false,\n", - " \"resume_from_checkpoint\": null,\n", - " \"hub_model_id\": null,\n", - " \"hub_strategy\": \"checkpoint\",\n", - " \"hub_token\": \"\",\n", - " \"hub_private_repo\": true,\n", - " \"hub_always_push\": false,\n", - " \"gradient_checkpointing\": false,\n", - " \"gradient_checkpointing_kwargs\": null,\n", - " \"include_inputs_for_metrics\": false,\n", - " \"fp16_backend\": \"auto\",\n", - " \"push_to_hub_model_id\": null,\n", - " \"push_to_hub_organization\": null,\n", - " \"push_to_hub_token\": \"\",\n", - " \"mp_parameters\": \"\",\n", - " \"auto_find_batch_size\": false,\n", - " \"full_determinism\": false,\n", - " \"torchdynamo\": null,\n", - " \"ray_scope\": \"last\",\n", - " \"ddp_timeout\": 1800,\n", - " \"torch_compile\": false,\n", - " \"torch_compile_backend\": null,\n", - " \"torch_compile_mode\": null,\n", - " \"dispatch_batches\": null,\n", - " \"split_batches\": false,\n", - " \"include_tokens_per_second\": false,\n", - " \"neftune_noise_alpha\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:06.725\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", - "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", - "loading file tokenizer.json from cache at None\n", - "loading file added_tokens.json from cache at None\n", - "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", - "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 15:59:07.023\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:07.025\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:07.028\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config is None. Model will be loaded using torch.float16\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", - "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "All model checkpoint weights were used when initializing OPTForCausalLM.\n", - "\n", - "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", - "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", - "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 15:59:23.485\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:23.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:23.628\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", - "Using auto half precision backend\n", - "\u001b[32m2023-11-14 15:59:23.882\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", - "\u001b[32m2023-11-14 15:59:23.885\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n" - ] - } - ] + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 15:59:06.695\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:06.699\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", + "{\n", + " \"experiment_key\": \"base\",\n", + " \"save_safetensors\": true,\n", + " \"max_shard_size\": \"10GB\",\n", + " \"local_rank\": 0,\n", + " \"use_gradient_checkpointing\": false,\n", + " \"trainer_key\": \"lm\",\n", + " \"force_fp32\": false,\n", + " \"force_fp16\": false,\n", + " \"from_gptq\": false,\n", + " \"huggingface_hub_token\": null,\n", + " \"deepspeed_stage\": 0,\n", + " \"deepspeed_config_path\": null,\n", + " \"fsdp_strategy\": \"\",\n", + " \"fsdp_offload\": true,\n", + " \"seed\": 42,\n", + " \"stabilize\": true,\n", + " \"path_to_env_file\": \"./.env\",\n", + " \"prepare_dataset\": true,\n", + " \"lora_hub_model_id\": null,\n", + " \"lora_model_local_path\": null,\n", + " \"fused_model_local_path\": null,\n", + " \"fuse_after_training\": false,\n", + " \"quantization_dataset_id\": null,\n", + " \"quantization_max_samples\": 1024,\n", + " \"quantized_model_path\": \"./quantized_model/\",\n", + " \"quantized_hub_model_id\": null,\n", + " \"quantized_hub_private_repo\": true,\n", + " \"dataset_key\": \"soda\",\n", + " \"train_local_path_to_data\": \"./train.jsonl\",\n", + " \"eval_local_path_to_data\": null,\n", + " \"shuffle\": true,\n", + " \"max_eval_samples\": 1000,\n", + " \"add_eval_to_train_if_no_path\": false,\n", + " \"tokenizer_name_or_path\": null,\n", + " \"tokenizer_use_fast\": null,\n", + " \"tokenizer_padding_side\": null,\n", + " \"collator_key\": \"lm\",\n", + " \"max_length\": 2048,\n", + " \"model_name_or_path\": \"facebook/opt-350m\",\n", + " \"push_to_hub_bos_add_bos_token\": false,\n", + " \"use_flash_attention_2\": false,\n", + " \"trust_remote_code\": false,\n", + " \"device_map\": null,\n", + " \"prepare_model_for_kbit_training\": true,\n", + " \"load_in_8bit\": false,\n", + " \"load_in_4bit\": false,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_quantize_after_model_init\": false,\n", + " \"gptq_bits\": 4,\n", + " \"gptq_group_size\": 128,\n", + " \"gptq_disable_exllama\": true,\n", + " \"apply_lora\": true,\n", + " \"lora_rank\": 8,\n", + " \"lora_alpha\": 32,\n", + " \"lora_dropout\": 0.1,\n", + " \"raw_lora_target_modules\": \"all\",\n", + " \"output_dir\": \"./outputs/\",\n", + " \"per_device_train_batch_size\": 2,\n", + " \"do_eval\": false,\n", + " \"per_device_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": null,\n", + " \"eval_delay\": 0,\n", + " \"eval_steps\": 1000,\n", + " \"warmup_steps\": 1000,\n", + " \"max_steps\": null,\n", + " \"num_train_epochs\": 1,\n", + " \"learning_rate\": 0.0002,\n", + " \"max_grad_norm\": 1.0,\n", + " \"weight_decay\": 0.001,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"logging_steps\": 10,\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"push_to_hub\": false,\n", + " \"hub_model_id\": null,\n", + " \"hub_private_repo\": true,\n", + " \"report_to_wandb\": false,\n", + " \"wandb_api_key\": null,\n", + " \"wandb_project\": null,\n", + " \"wandb_entity\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:06.700\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:06.709\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", + "PyTorch: setting up devices\n", + "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", + "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", + "\u001b[32m2023-11-14 15:59:06.722\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", + "{\n", + " \"output_dir\": \"./outputs/\",\n", + " \"overwrite_output_dir\": false,\n", + " \"do_train\": false,\n", + " \"do_eval\": false,\n", + " \"do_predict\": false,\n", + " \"evaluation_strategy\": \"no\",\n", + " \"prediction_loss_only\": false,\n", + " \"per_device_train_batch_size\": 2,\n", + " \"per_device_eval_batch_size\": 2,\n", + " \"per_gpu_train_batch_size\": null,\n", + " \"per_gpu_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": 1,\n", + " \"eval_delay\": 0,\n", + " \"learning_rate\": 0.0002,\n", + " \"weight_decay\": 0.001,\n", + " \"adam_beta1\": 0.9,\n", + " \"adam_beta2\": 0.999,\n", + " \"adam_epsilon\": 1e-08,\n", + " \"max_grad_norm\": 1.0,\n", + " \"num_train_epochs\": 1,\n", + " \"max_steps\": -1,\n", + " \"lr_scheduler_type\": \"linear\",\n", + " \"warmup_ratio\": 0.0,\n", + " \"warmup_steps\": 1000,\n", + " \"log_level\": \"info\",\n", + " \"log_level_replica\": \"warning\",\n", + " \"log_on_each_node\": true,\n", + " \"logging_dir\": \"./outputs/runs/Nov14_15-59-06_735f762378cc\",\n", + " \"logging_strategy\": \"steps\",\n", + " \"logging_first_step\": true,\n", + " \"logging_steps\": 10,\n", + " \"logging_nan_inf_filter\": true,\n", + " \"save_strategy\": \"steps\",\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"save_safetensors\": true,\n", + " \"save_on_each_node\": false,\n", + " \"no_cuda\": false,\n", + " \"use_cpu\": false,\n", + " \"use_mps_device\": false,\n", + " \"seed\": 42,\n", + " \"data_seed\": 42,\n", + " \"jit_mode_eval\": false,\n", + " \"use_ipex\": false,\n", + " \"bf16\": false,\n", + " \"fp16\": true,\n", + " \"fp16_opt_level\": \"O1\",\n", + " \"half_precision_backend\": \"auto\",\n", + " \"bf16_full_eval\": false,\n", + " \"fp16_full_eval\": false,\n", + " \"tf32\": null,\n", + " \"local_rank\": 0,\n", + " \"ddp_backend\": null,\n", + " \"tpu_num_cores\": null,\n", + " \"tpu_metrics_debug\": false,\n", + " \"debug\": [],\n", + " \"dataloader_drop_last\": false,\n", + " \"eval_steps\": 1000,\n", + " \"dataloader_num_workers\": 0,\n", + " \"past_index\": -1,\n", + " \"run_name\": \"./outputs/\",\n", + " \"disable_tqdm\": false,\n", + " \"remove_unused_columns\": false,\n", + " \"label_names\": null,\n", + " \"load_best_model_at_end\": false,\n", + " \"metric_for_best_model\": \"loss\",\n", + " \"greater_is_better\": false,\n", + " \"ignore_data_skip\": false,\n", + " \"fsdp\": [],\n", + " \"fsdp_min_num_params\": 0,\n", + " \"fsdp_config\": {\n", + " \"min_num_params\": 0,\n", + " \"xla\": false,\n", + " \"xla_fsdp_grad_ckpt\": false\n", + " },\n", + " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", + " \"deepspeed\": null,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"optim_args\": null,\n", + " \"adafactor\": false,\n", + " \"group_by_length\": false,\n", + " \"length_column_name\": \"length\",\n", + " \"report_to\": [\n", + " \"tensorboard\"\n", + " ],\n", + " \"ddp_find_unused_parameters\": null,\n", + " \"ddp_bucket_cap_mb\": null,\n", + " \"ddp_broadcast_buffers\": null,\n", + " \"dataloader_pin_memory\": true,\n", + " \"skip_memory_metrics\": true,\n", + " \"use_legacy_prediction_loop\": false,\n", + " \"push_to_hub\": false,\n", + " \"resume_from_checkpoint\": null,\n", + " \"hub_model_id\": null,\n", + " \"hub_strategy\": \"checkpoint\",\n", + " \"hub_token\": \"\",\n", + " \"hub_private_repo\": true,\n", + " \"hub_always_push\": false,\n", + " \"gradient_checkpointing\": false,\n", + " \"gradient_checkpointing_kwargs\": null,\n", + " \"include_inputs_for_metrics\": false,\n", + " \"fp16_backend\": \"auto\",\n", + " \"push_to_hub_model_id\": null,\n", + " \"push_to_hub_organization\": null,\n", + " \"push_to_hub_token\": \"\",\n", + " \"mp_parameters\": \"\",\n", + " \"auto_find_batch_size\": false,\n", + " \"full_determinism\": false,\n", + " \"torchdynamo\": null,\n", + " \"ray_scope\": \"last\",\n", + " \"ddp_timeout\": 1800,\n", + " \"torch_compile\": false,\n", + " \"torch_compile_backend\": null,\n", + " \"torch_compile_mode\": null,\n", + " \"dispatch_batches\": null,\n", + " \"split_batches\": false,\n", + " \"include_tokens_per_second\": false,\n", + " \"neftune_noise_alpha\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:06.725\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", + "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", + "loading file tokenizer.json from cache at None\n", + "loading file added_tokens.json from cache at None\n", + "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", + "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 15:59:07.023\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:07.025\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:07.028\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config is None. Model will be loaded using torch.float16\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", + "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "All model checkpoint weights were used when initializing OPTForCausalLM.\n", + "\n", + "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", + "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 15:59:23.485\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:23.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:23.628\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", + "Using auto half precision backend\n", + "\u001b[32m2023-11-14 15:59:23.882\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", + "\u001b[32m2023-11-14 15:59:23.885\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## 🚄 Run experiment" + ], + "metadata": { + "id": "1WFN6ISmJnFm" + } + }, + { + "cell_type": "code", + "source": [ + "experiment.run()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 881 }, + "id": "rzPdKTYW6_yF", + "outputId": "8ce3719f-0907-4517-8717-2dc4d156af38" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "## 🚄 Run experiment" - ], - "metadata": { - "id": "1WFN6ISmJnFm" - } + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 15:59:23.895\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", + "***** Running training *****\n", + " Num examples = 300\n", + " Num Epochs = 1\n", + " Instantaneous batch size per device = 2\n", + " Total train batch size (w. parallel, distributed & accumulation) = 2\n", + " Gradient Accumulation steps = 1\n", + " Total optimization steps = 150\n", + " Number of trainable parameters = 3,563,520\n" + ] }, { - "cell_type": "code", - "source": [ - "experiment.run()" + "output_type": "display_data", + "data": { + "text/plain": [ + "" ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 881 - }, - "id": "rzPdKTYW6_yF", - "outputId": "8ce3719f-0907-4517-8717-2dc4d156af38" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 15:59:23.895\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", - "***** Running training *****\n", - " Num examples = 300\n", - " Num Epochs = 1\n", - " Instantaneous batch size per device = 2\n", - " Total train batch size (w. parallel, distributed & accumulation) = 2\n", - " Gradient Accumulation steps = 1\n", - " Total optimization steps = 150\n", - " Number of trainable parameters = 3,563,520\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "

\n", - " \n", - " \n", - " [150/150 00:48, Epoch 1/1]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining Loss
13.626900
103.827700
203.748600
303.797700
403.303100
502.994300
602.499000
702.567500
801.788300
901.186500
1000.937400
1100.693600
1200.632500
1300.561100
1400.453600
1500.473700

" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Saving model checkpoint to ./outputs/checkpoint-100\n", - "\n", - "\n", - "Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\n", - "\n", - "\u001b[32m2023-11-14 16:00:12.883\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:12.888\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" - ] - } + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [150/150 00:48, Epoch 1/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
13.626900
103.827700
203.748600
303.797700
403.303100
502.994300
602.499000
702.567500
801.788300
901.186500
1000.937400
1100.693600
1200.632500
1300.561100
1400.453600
1500.473700

" ] + }, + "metadata": {} }, { - "cell_type": "markdown", - "source": [ - "## 🎉 Done!\n", - "\n", - "You are trained a model using `xllm`" - ], - "metadata": { - "id": "gU3Dy3FlKoCg" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Fuse model" - ], - "metadata": { - "id": "UPo9UQuptnuu" - } - }, - { - "cell_type": "code", - "source": [ - "# experiment.fuse_lora()" - ], - "metadata": { - "id": "r__1558mtngt" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Get the model" - ], - "metadata": { - "id": "BvihxRPHKuGd" - } - }, - { - "cell_type": "code", - "source": [ - "# experiment.model" - ], - "metadata": { - "id": "63ibIRNQLiTS" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## You can save the model\n", - "\n", - "If you have not fuse the model, then only the LoRA weights will be saved." - ], - "metadata": { - "id": "sfzXFLW2Lnpd" - } - }, - { - "cell_type": "code", - "source": [ - "# experiment.model.save_pretrained(\"./trained_model/\")" - ], - "metadata": { - "id": "wj6oheYCLkv5" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## You could push the model to the HuggingFace Hub\n", - "\n", - "If you have not fuse the model, then only the LoRA weights will be saved.\n", - "\n", - "Make sure you are logged in HuggingFace Hub. You can run this command:\n", - "\n", - "```python\n", - "!huggingface-cli login\n", - "```\n", - "\n", - "Or you can set the environment variable with your Access token. You can find your token here: https://huggingface.co/settings/tokens\n", - "\n", - "```\n", - "import os\n", - "\n", - "os.environ[\"HUGGING_FACE_HUB_TOKEN\"] = \"YOUR_ACCESS_TOKEN\"\n", - "```" - ], - "metadata": { - "id": "jgkDeErVLq-H" - } - }, - { - "cell_type": "code", - "source": [ - "# push the model and the tokenizer to the HuggingFace Hub\n", - "# experiment.push_to_hub(\n", - "# repo_id=\"YOUR_LOGIN_AT_HF_HUB/MODEL_NAME\",\n", - "# private=False,\n", - "# safe_serialization=True\n", - "# )" - ], - "metadata": { - "id": "_jhP6AG_MIWJ" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## 🎉 Done!\n", - "\n", - "You've trained the model using `xllm` and uploaded it to the hub" - ], - "metadata": { - "id": "0nBPNvljMhQx" - } - }, - { - "cell_type": "markdown", - "source": [ - "# Add QLoRA\n", - "\n", - "To train the `QLoRA` model, we need to load the backbone model using `bitsandbytes` library and int4 (or int8) weights." - ], - "metadata": { - "id": "i0cnsI3Quizi" - } - }, - { - "cell_type": "code", - "source": [ - "# config with QLoRA\n", - "config = Config(\n", - " model_name_or_path=\"facebook/opt-350m\",\n", - " stabilize=True,\n", - " apply_lora=True,\n", - " load_in_4bit=True,\n", - " prepare_model_for_kbit_training=True,\n", - ")" - ], - "metadata": { - "id": "Jwj6AQjWvCPA" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### You can explicitly specify the values of additional parameters in bitsandbytes quantization" - ], - "metadata": { - "id": "MKya31NSvMHu" - } - }, - { - "cell_type": "code", - "source": [ - "# # extended config with QLoRA\n", - "# config = Config(\n", - "# model_name_or_path=\"facebook/opt-350m\",\n", - "# stabilize=True,\n", - "# apply_lora=True,\n", - "# load_in_4bit=True,\n", - "# prepare_model_for_kbit_training=True,\n", - "# llm_int8_threshold=6.0,\n", - "# llm_int8_has_fp16_weight=True,\n", - "# bnb_4bit_use_double_quant=True,\n", - "# bnb_4bit_quant_type=\"nf4\",\n", - "# )" - ], - "metadata": { - "id": "5Qq9t9VfvPg9" - }, - "execution_count": null, - "outputs": [] + "output_type": "stream", + "name": "stderr", + "text": [ + "Saving model checkpoint to ./outputs/checkpoint-100\n", + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "\u001b[32m2023-11-14 16:00:12.883\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:12.888\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## 🎉 Done!\n", + "\n", + "You are trained a model using `xllm`" + ], + "metadata": { + "id": "gU3Dy3FlKoCg" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Fuse model" + ], + "metadata": { + "id": "UPo9UQuptnuu" + } + }, + { + "cell_type": "code", + "source": [ + "# experiment.fuse_lora()" + ], + "metadata": { + "id": "r__1558mtngt" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Get the model" + ], + "metadata": { + "id": "BvihxRPHKuGd" + } + }, + { + "cell_type": "code", + "source": [ + "# experiment.model" + ], + "metadata": { + "id": "63ibIRNQLiTS" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## You can save the model\n", + "\n", + "If you have not fuse the model, then only the LoRA weights will be saved." + ], + "metadata": { + "id": "sfzXFLW2Lnpd" + } + }, + { + "cell_type": "code", + "source": [ + "# experiment.model.save_pretrained(\"./trained_model/\")" + ], + "metadata": { + "id": "wj6oheYCLkv5" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## You could push the model to the HuggingFace Hub\n", + "\n", + "If you have not fuse the model, then only the LoRA weights will be saved.\n", + "\n", + "Make sure you are logged in HuggingFace Hub. You can run this command:\n", + "\n", + "```python\n", + "!huggingface-cli login\n", + "```\n", + "\n", + "Or you can set the environment variable with your Access token. You can find your token here: https://huggingface.co/settings/tokens\n", + "\n", + "```\n", + "import os\n", + "\n", + "os.environ[\"HUGGING_FACE_HUB_TOKEN\"] = \"YOUR_ACCESS_TOKEN\"\n", + "```" + ], + "metadata": { + "id": "jgkDeErVLq-H" + } + }, + { + "cell_type": "code", + "source": [ + "# push the model and the tokenizer to the HuggingFace Hub\n", + "# experiment.push_to_hub(\n", + "# repo_id=\"YOUR_LOGIN_AT_HF_HUB/MODEL_NAME\",\n", + "# private=False,\n", + "# safe_serialization=True\n", + "# )" + ], + "metadata": { + "id": "_jhP6AG_MIWJ" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## 🎉 Done!\n", + "\n", + "You've trained the model using `xllm` and uploaded it to the hub" + ], + "metadata": { + "id": "0nBPNvljMhQx" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Add QLoRA\n", + "\n", + "To train the `QLoRA` model, we need to load the backbone model using `bitsandbytes` library and int4 (or int8) weights." + ], + "metadata": { + "id": "i0cnsI3Quizi" + } + }, + { + "cell_type": "code", + "source": [ + "# config with QLoRA\n", + "config = Config(\n", + " model_name_or_path=\"facebook/opt-350m\",\n", + " stabilize=True,\n", + " apply_lora=True,\n", + " load_in_4bit=True,\n", + " prepare_model_for_kbit_training=True,\n", + ")" + ], + "metadata": { + "id": "Jwj6AQjWvCPA" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### You can explicitly specify the values of additional parameters in bitsandbytes quantization" + ], + "metadata": { + "id": "MKya31NSvMHu" + } + }, + { + "cell_type": "code", + "source": [ + "# # extended config with QLoRA\n", + "# config = Config(\n", + "# model_name_or_path=\"facebook/opt-350m\",\n", + "# stabilize=True,\n", + "# apply_lora=True,\n", + "# load_in_4bit=True,\n", + "# prepare_model_for_kbit_training=True,\n", + "# llm_int8_threshold=6.0,\n", + "# llm_int8_has_fp16_weight=True,\n", + "# bnb_4bit_use_double_quant=True,\n", + "# bnb_4bit_quant_type=\"nf4\",\n", + "# )" + ], + "metadata": { + "id": "5Qq9t9VfvPg9" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## All other steps are the same" + ], + "metadata": { + "id": "jY7omBtEv2V6" + } + }, + { + "cell_type": "code", + "source": [ + "train_data = [\"Hello!\", \"How are you?\", \"Are you okay?\"] * 100\n", + "train_dataset = GeneralDataset.from_list(data=train_data)\n", + "experiment = Experiment(config=config, train_dataset=train_dataset)\n", + "experiment.build()\n", + "experiment.run()\n", + "# experiment.fuse_lora()" + ], + "metadata": { + "id": "FWhYuHl8v1xr", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 }, + "outputId": "1f69aec1-d76e-4756-e9af-74d74bac532e" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "## All other steps are the same" - ], - "metadata": { - "id": "jY7omBtEv2V6" - } + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 16:00:12.955\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:12.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", + "{\n", + " \"experiment_key\": \"base\",\n", + " \"save_safetensors\": true,\n", + " \"max_shard_size\": \"10GB\",\n", + " \"local_rank\": 0,\n", + " \"use_gradient_checkpointing\": false,\n", + " \"trainer_key\": \"lm\",\n", + " \"force_fp32\": false,\n", + " \"force_fp16\": false,\n", + " \"from_gptq\": false,\n", + " \"huggingface_hub_token\": null,\n", + " \"deepspeed_stage\": 0,\n", + " \"deepspeed_config_path\": null,\n", + " \"fsdp_strategy\": \"\",\n", + " \"fsdp_offload\": true,\n", + " \"seed\": 42,\n", + " \"stabilize\": true,\n", + " \"path_to_env_file\": \"./.env\",\n", + " \"prepare_dataset\": true,\n", + " \"lora_hub_model_id\": null,\n", + " \"lora_model_local_path\": null,\n", + " \"fused_model_local_path\": null,\n", + " \"fuse_after_training\": false,\n", + " \"quantization_dataset_id\": null,\n", + " \"quantization_max_samples\": 1024,\n", + " \"quantized_model_path\": \"./quantized_model/\",\n", + " \"quantized_hub_model_id\": null,\n", + " \"quantized_hub_private_repo\": true,\n", + " \"dataset_key\": \"soda\",\n", + " \"train_local_path_to_data\": \"./train.jsonl\",\n", + " \"eval_local_path_to_data\": null,\n", + " \"shuffle\": true,\n", + " \"max_eval_samples\": 1000,\n", + " \"add_eval_to_train_if_no_path\": false,\n", + " \"tokenizer_name_or_path\": null,\n", + " \"tokenizer_use_fast\": null,\n", + " \"tokenizer_padding_side\": null,\n", + " \"collator_key\": \"lm\",\n", + " \"max_length\": 2048,\n", + " \"model_name_or_path\": \"facebook/opt-350m\",\n", + " \"push_to_hub_bos_add_bos_token\": false,\n", + " \"use_flash_attention_2\": false,\n", + " \"trust_remote_code\": false,\n", + " \"device_map\": null,\n", + " \"prepare_model_for_kbit_training\": true,\n", + " \"load_in_8bit\": false,\n", + " \"load_in_4bit\": true,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_quantize_after_model_init\": false,\n", + " \"gptq_bits\": 4,\n", + " \"gptq_group_size\": 128,\n", + " \"gptq_disable_exllama\": true,\n", + " \"apply_lora\": true,\n", + " \"lora_rank\": 8,\n", + " \"lora_alpha\": 32,\n", + " \"lora_dropout\": 0.1,\n", + " \"raw_lora_target_modules\": \"all\",\n", + " \"output_dir\": \"./outputs/\",\n", + " \"per_device_train_batch_size\": 2,\n", + " \"do_eval\": false,\n", + " \"per_device_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": null,\n", + " \"eval_delay\": 0,\n", + " \"eval_steps\": 1000,\n", + " \"warmup_steps\": 1000,\n", + " \"max_steps\": null,\n", + " \"num_train_epochs\": 1,\n", + " \"learning_rate\": 0.0002,\n", + " \"max_grad_norm\": 1.0,\n", + " \"weight_decay\": 0.001,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"logging_steps\": 10,\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"push_to_hub\": false,\n", + " \"hub_model_id\": null,\n", + " \"hub_private_repo\": true,\n", + " \"report_to_wandb\": false,\n", + " \"wandb_api_key\": null,\n", + " \"wandb_project\": null,\n", + " \"wandb_entity\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:12.960\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:12.962\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", + "PyTorch: setting up devices\n", + "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", + "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", + "\u001b[32m2023-11-14 16:00:12.973\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", + "{\n", + " \"output_dir\": \"./outputs/\",\n", + " \"overwrite_output_dir\": false,\n", + " \"do_train\": false,\n", + " \"do_eval\": false,\n", + " \"do_predict\": false,\n", + " \"evaluation_strategy\": \"no\",\n", + " \"prediction_loss_only\": false,\n", + " \"per_device_train_batch_size\": 2,\n", + " \"per_device_eval_batch_size\": 2,\n", + " \"per_gpu_train_batch_size\": null,\n", + " \"per_gpu_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": 1,\n", + " \"eval_delay\": 0,\n", + " \"learning_rate\": 0.0002,\n", + " \"weight_decay\": 0.001,\n", + " \"adam_beta1\": 0.9,\n", + " \"adam_beta2\": 0.999,\n", + " \"adam_epsilon\": 1e-08,\n", + " \"max_grad_norm\": 1.0,\n", + " \"num_train_epochs\": 1,\n", + " \"max_steps\": -1,\n", + " \"lr_scheduler_type\": \"linear\",\n", + " \"warmup_ratio\": 0.0,\n", + " \"warmup_steps\": 1000,\n", + " \"log_level\": \"info\",\n", + " \"log_level_replica\": \"warning\",\n", + " \"log_on_each_node\": true,\n", + " \"logging_dir\": \"./outputs/runs/Nov14_16-00-12_735f762378cc\",\n", + " \"logging_strategy\": \"steps\",\n", + " \"logging_first_step\": true,\n", + " \"logging_steps\": 10,\n", + " \"logging_nan_inf_filter\": true,\n", + " \"save_strategy\": \"steps\",\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"save_safetensors\": true,\n", + " \"save_on_each_node\": false,\n", + " \"no_cuda\": false,\n", + " \"use_cpu\": false,\n", + " \"use_mps_device\": false,\n", + " \"seed\": 42,\n", + " \"data_seed\": 42,\n", + " \"jit_mode_eval\": false,\n", + " \"use_ipex\": false,\n", + " \"bf16\": false,\n", + " \"fp16\": true,\n", + " \"fp16_opt_level\": \"O1\",\n", + " \"half_precision_backend\": \"auto\",\n", + " \"bf16_full_eval\": false,\n", + " \"fp16_full_eval\": false,\n", + " \"tf32\": null,\n", + " \"local_rank\": 0,\n", + " \"ddp_backend\": null,\n", + " \"tpu_num_cores\": null,\n", + " \"tpu_metrics_debug\": false,\n", + " \"debug\": [],\n", + " \"dataloader_drop_last\": false,\n", + " \"eval_steps\": 1000,\n", + " \"dataloader_num_workers\": 0,\n", + " \"past_index\": -1,\n", + " \"run_name\": \"./outputs/\",\n", + " \"disable_tqdm\": false,\n", + " \"remove_unused_columns\": false,\n", + " \"label_names\": null,\n", + " \"load_best_model_at_end\": false,\n", + " \"metric_for_best_model\": \"loss\",\n", + " \"greater_is_better\": false,\n", + " \"ignore_data_skip\": false,\n", + " \"fsdp\": [],\n", + " \"fsdp_min_num_params\": 0,\n", + " \"fsdp_config\": {\n", + " \"min_num_params\": 0,\n", + " \"xla\": false,\n", + " \"xla_fsdp_grad_ckpt\": false\n", + " },\n", + " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", + " \"deepspeed\": null,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"optim_args\": null,\n", + " \"adafactor\": false,\n", + " \"group_by_length\": false,\n", + " \"length_column_name\": \"length\",\n", + " \"report_to\": [\n", + " \"tensorboard\"\n", + " ],\n", + " \"ddp_find_unused_parameters\": null,\n", + " \"ddp_bucket_cap_mb\": null,\n", + " \"ddp_broadcast_buffers\": null,\n", + " \"dataloader_pin_memory\": true,\n", + " \"skip_memory_metrics\": true,\n", + " \"use_legacy_prediction_loop\": false,\n", + " \"push_to_hub\": false,\n", + " \"resume_from_checkpoint\": null,\n", + " \"hub_model_id\": null,\n", + " \"hub_strategy\": \"checkpoint\",\n", + " \"hub_token\": \"\",\n", + " \"hub_private_repo\": true,\n", + " \"hub_always_push\": false,\n", + " \"gradient_checkpointing\": false,\n", + " \"gradient_checkpointing_kwargs\": null,\n", + " \"include_inputs_for_metrics\": false,\n", + " \"fp16_backend\": \"auto\",\n", + " \"push_to_hub_model_id\": null,\n", + " \"push_to_hub_organization\": null,\n", + " \"push_to_hub_token\": \"\",\n", + " \"mp_parameters\": \"\",\n", + " \"auto_find_batch_size\": false,\n", + " \"full_determinism\": false,\n", + " \"torchdynamo\": null,\n", + " \"ray_scope\": \"last\",\n", + " \"ddp_timeout\": 1800,\n", + " \"torch_compile\": false,\n", + " \"torch_compile_backend\": null,\n", + " \"torch_compile_mode\": null,\n", + " \"dispatch_batches\": null,\n", + " \"split_batches\": false,\n", + " \"include_tokens_per_second\": false,\n", + " \"neftune_noise_alpha\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:12.977\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", + "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", + "loading file tokenizer.json from cache at None\n", + "loading file added_tokens.json from cache at None\n", + "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", + "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 16:00:13.378\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:13.380\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:13.386\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config was built:\n", + "{\n", + " \"bnb_4bit_compute_dtype\": \"float16\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"load_in_4bit\": true\n", + "}\n", + "\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "The device_map was not initialized. Setting device_map to {'':torch.cuda.current_device()}. If you want to use the model for inference, please set device_map ='auto' \n", + "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", + "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "Detected 4-bit loading: activating 4-bit loading for this model\n", + "All model checkpoint weights were used when initializing OPTForCausalLM.\n", + "\n", + "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", + "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 16:00:15.797\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel prepared for kbit training. Gradient checkpointing: False\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:15.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:15.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:15.966\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", + "Using auto half precision backend\n", + "\u001b[32m2023-11-14 16:00:15.976\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:15.977\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n", + "\u001b[32m2023-11-14 16:00:15.981\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", + "***** Running training *****\n", + " Num examples = 300\n", + " Num Epochs = 1\n", + " Instantaneous batch size per device = 2\n", + " Total train batch size (w. parallel, distributed & accumulation) = 2\n", + " Gradient Accumulation steps = 1\n", + " Total optimization steps = 150\n", + " Number of trainable parameters = 3,563,520\n" + ] }, { - "cell_type": "code", - "source": [ - "train_data = [\"Hello!\", \"How are you?\", \"Are you okay?\"] * 100\n", - "train_dataset = GeneralDataset.from_list(data=train_data)\n", - "experiment = Experiment(config=config, train_dataset=train_dataset)\n", - "experiment.build()\n", - "experiment.run()\n", - "# experiment.fuse_lora()" + "output_type": "display_data", + "data": { + "text/plain": [ + "" ], - "metadata": { - "id": "FWhYuHl8v1xr", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "outputId": "1f69aec1-d76e-4756-e9af-74d74bac532e" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 16:00:12.955\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:12.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", - "{\n", - " \"experiment_key\": \"base\",\n", - " \"save_safetensors\": true,\n", - " \"max_shard_size\": \"10GB\",\n", - " \"local_rank\": 0,\n", - " \"use_gradient_checkpointing\": false,\n", - " \"trainer_key\": \"lm\",\n", - " \"force_fp32\": false,\n", - " \"force_fp16\": false,\n", - " \"from_gptq\": false,\n", - " \"huggingface_hub_token\": null,\n", - " \"deepspeed_stage\": 0,\n", - " \"deepspeed_config_path\": null,\n", - " \"fsdp_strategy\": \"\",\n", - " \"fsdp_offload\": true,\n", - " \"seed\": 42,\n", - " \"stabilize\": true,\n", - " \"path_to_env_file\": \"./.env\",\n", - " \"prepare_dataset\": true,\n", - " \"lora_hub_model_id\": null,\n", - " \"lora_model_local_path\": null,\n", - " \"fused_model_local_path\": null,\n", - " \"fuse_after_training\": false,\n", - " \"quantization_dataset_id\": null,\n", - " \"quantization_max_samples\": 1024,\n", - " \"quantized_model_path\": \"./quantized_model/\",\n", - " \"quantized_hub_model_id\": null,\n", - " \"quantized_hub_private_repo\": true,\n", - " \"dataset_key\": \"soda\",\n", - " \"train_local_path_to_data\": \"./train.jsonl\",\n", - " \"eval_local_path_to_data\": null,\n", - " \"shuffle\": true,\n", - " \"max_eval_samples\": 1000,\n", - " \"add_eval_to_train_if_no_path\": false,\n", - " \"tokenizer_name_or_path\": null,\n", - " \"tokenizer_use_fast\": null,\n", - " \"tokenizer_padding_side\": null,\n", - " \"collator_key\": \"lm\",\n", - " \"max_length\": 2048,\n", - " \"model_name_or_path\": \"facebook/opt-350m\",\n", - " \"push_to_hub_bos_add_bos_token\": false,\n", - " \"use_flash_attention_2\": false,\n", - " \"trust_remote_code\": false,\n", - " \"device_map\": null,\n", - " \"prepare_model_for_kbit_training\": true,\n", - " \"load_in_8bit\": false,\n", - " \"load_in_4bit\": true,\n", - " \"llm_int8_threshold\": 6.0,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_quantize_after_model_init\": false,\n", - " \"gptq_bits\": 4,\n", - " \"gptq_group_size\": 128,\n", - " \"gptq_disable_exllama\": true,\n", - " \"apply_lora\": true,\n", - " \"lora_rank\": 8,\n", - " \"lora_alpha\": 32,\n", - " \"lora_dropout\": 0.1,\n", - " \"raw_lora_target_modules\": \"all\",\n", - " \"output_dir\": \"./outputs/\",\n", - " \"per_device_train_batch_size\": 2,\n", - " \"do_eval\": false,\n", - " \"per_device_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": null,\n", - " \"eval_delay\": 0,\n", - " \"eval_steps\": 1000,\n", - " \"warmup_steps\": 1000,\n", - " \"max_steps\": null,\n", - " \"num_train_epochs\": 1,\n", - " \"learning_rate\": 0.0002,\n", - " \"max_grad_norm\": 1.0,\n", - " \"weight_decay\": 0.001,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"logging_steps\": 10,\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"push_to_hub\": false,\n", - " \"hub_model_id\": null,\n", - " \"hub_private_repo\": true,\n", - " \"report_to_wandb\": false,\n", - " \"wandb_api_key\": null,\n", - " \"wandb_project\": null,\n", - " \"wandb_entity\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:12.960\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:12.962\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", - "PyTorch: setting up devices\n", - "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", - "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", - "\u001b[32m2023-11-14 16:00:12.973\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", - "{\n", - " \"output_dir\": \"./outputs/\",\n", - " \"overwrite_output_dir\": false,\n", - " \"do_train\": false,\n", - " \"do_eval\": false,\n", - " \"do_predict\": false,\n", - " \"evaluation_strategy\": \"no\",\n", - " \"prediction_loss_only\": false,\n", - " \"per_device_train_batch_size\": 2,\n", - " \"per_device_eval_batch_size\": 2,\n", - " \"per_gpu_train_batch_size\": null,\n", - " \"per_gpu_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": 1,\n", - " \"eval_delay\": 0,\n", - " \"learning_rate\": 0.0002,\n", - " \"weight_decay\": 0.001,\n", - " \"adam_beta1\": 0.9,\n", - " \"adam_beta2\": 0.999,\n", - " \"adam_epsilon\": 1e-08,\n", - " \"max_grad_norm\": 1.0,\n", - " \"num_train_epochs\": 1,\n", - " \"max_steps\": -1,\n", - " \"lr_scheduler_type\": \"linear\",\n", - " \"warmup_ratio\": 0.0,\n", - " \"warmup_steps\": 1000,\n", - " \"log_level\": \"info\",\n", - " \"log_level_replica\": \"warning\",\n", - " \"log_on_each_node\": true,\n", - " \"logging_dir\": \"./outputs/runs/Nov14_16-00-12_735f762378cc\",\n", - " \"logging_strategy\": \"steps\",\n", - " \"logging_first_step\": true,\n", - " \"logging_steps\": 10,\n", - " \"logging_nan_inf_filter\": true,\n", - " \"save_strategy\": \"steps\",\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"save_safetensors\": true,\n", - " \"save_on_each_node\": false,\n", - " \"no_cuda\": false,\n", - " \"use_cpu\": false,\n", - " \"use_mps_device\": false,\n", - " \"seed\": 42,\n", - " \"data_seed\": 42,\n", - " \"jit_mode_eval\": false,\n", - " \"use_ipex\": false,\n", - " \"bf16\": false,\n", - " \"fp16\": true,\n", - " \"fp16_opt_level\": \"O1\",\n", - " \"half_precision_backend\": \"auto\",\n", - " \"bf16_full_eval\": false,\n", - " \"fp16_full_eval\": false,\n", - " \"tf32\": null,\n", - " \"local_rank\": 0,\n", - " \"ddp_backend\": null,\n", - " \"tpu_num_cores\": null,\n", - " \"tpu_metrics_debug\": false,\n", - " \"debug\": [],\n", - " \"dataloader_drop_last\": false,\n", - " \"eval_steps\": 1000,\n", - " \"dataloader_num_workers\": 0,\n", - " \"past_index\": -1,\n", - " \"run_name\": \"./outputs/\",\n", - " \"disable_tqdm\": false,\n", - " \"remove_unused_columns\": false,\n", - " \"label_names\": null,\n", - " \"load_best_model_at_end\": false,\n", - " \"metric_for_best_model\": \"loss\",\n", - " \"greater_is_better\": false,\n", - " \"ignore_data_skip\": false,\n", - " \"fsdp\": [],\n", - " \"fsdp_min_num_params\": 0,\n", - " \"fsdp_config\": {\n", - " \"min_num_params\": 0,\n", - " \"xla\": false,\n", - " \"xla_fsdp_grad_ckpt\": false\n", - " },\n", - " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", - " \"deepspeed\": null,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"optim_args\": null,\n", - " \"adafactor\": false,\n", - " \"group_by_length\": false,\n", - " \"length_column_name\": \"length\",\n", - " \"report_to\": [\n", - " \"tensorboard\"\n", - " ],\n", - " \"ddp_find_unused_parameters\": null,\n", - " \"ddp_bucket_cap_mb\": null,\n", - " \"ddp_broadcast_buffers\": null,\n", - " \"dataloader_pin_memory\": true,\n", - " \"skip_memory_metrics\": true,\n", - " \"use_legacy_prediction_loop\": false,\n", - " \"push_to_hub\": false,\n", - " \"resume_from_checkpoint\": null,\n", - " \"hub_model_id\": null,\n", - " \"hub_strategy\": \"checkpoint\",\n", - " \"hub_token\": \"\",\n", - " \"hub_private_repo\": true,\n", - " \"hub_always_push\": false,\n", - " \"gradient_checkpointing\": false,\n", - " \"gradient_checkpointing_kwargs\": null,\n", - " \"include_inputs_for_metrics\": false,\n", - " \"fp16_backend\": \"auto\",\n", - " \"push_to_hub_model_id\": null,\n", - " \"push_to_hub_organization\": null,\n", - " \"push_to_hub_token\": \"\",\n", - " \"mp_parameters\": \"\",\n", - " \"auto_find_batch_size\": false,\n", - " \"full_determinism\": false,\n", - " \"torchdynamo\": null,\n", - " \"ray_scope\": \"last\",\n", - " \"ddp_timeout\": 1800,\n", - " \"torch_compile\": false,\n", - " \"torch_compile_backend\": null,\n", - " \"torch_compile_mode\": null,\n", - " \"dispatch_batches\": null,\n", - " \"split_batches\": false,\n", - " \"include_tokens_per_second\": false,\n", - " \"neftune_noise_alpha\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:12.977\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mEval dataset is None\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", - "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", - "loading file tokenizer.json from cache at None\n", - "loading file added_tokens.json from cache at None\n", - "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", - "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 16:00:13.378\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:13.380\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:13.386\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config was built:\n", - "{\n", - " \"bnb_4bit_compute_dtype\": \"float16\",\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"load_in_4bit\": true\n", - "}\n", - "\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "The device_map was not initialized. Setting device_map to {'':torch.cuda.current_device()}. If you want to use the model for inference, please set device_map ='auto' \n", - "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", - "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "Detected 4-bit loading: activating 4-bit loading for this model\n", - "All model checkpoint weights were used when initializing OPTForCausalLM.\n", - "\n", - "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", - "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", - "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 16:00:15.797\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel prepared for kbit training. Gradient checkpointing: False\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:15.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:15.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:15.966\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", - "Using auto half precision backend\n", - "\u001b[32m2023-11-14 16:00:15.976\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:15.977\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n", - "\u001b[32m2023-11-14 16:00:15.981\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", - "***** Running training *****\n", - " Num examples = 300\n", - " Num Epochs = 1\n", - " Instantaneous batch size per device = 2\n", - " Total train batch size (w. parallel, distributed & accumulation) = 2\n", - " Gradient Accumulation steps = 1\n", - " Total optimization steps = 150\n", - " Number of trainable parameters = 3,563,520\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "

\n", - " \n", - " \n", - " [150/150 00:53, Epoch 1/1]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining Loss
14.945300
104.768000
204.720400
304.408000
403.776000
503.216700
602.629700
702.522100
801.713300
901.168800
1000.829100
1100.735500
1200.586600
1300.437500
1400.422200
1500.428500

" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Saving model checkpoint to ./outputs/checkpoint-100\n", - "\n", - "\n", - "Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\n", - "\n", - "\u001b[32m2023-11-14 16:01:09.884\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:09.888\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" - ] - } + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [150/150 00:53, Epoch 1/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
14.945300
104.768000
204.720400
304.408000
403.776000
503.216700
602.629700
702.522100
801.713300
901.168800
1000.829100
1100.735500
1200.586600
1300.437500
1400.422200
1500.428500

" ] + }, + "metadata": {} }, { - "cell_type": "markdown", - "source": [ - "## You also can add `Gradient Checkpointing`\n", - "\n", - "This will help to use `less GPU memory` during training, that is, you will be able to learn more than without this technique. The disadvantages of this technique is slowing down the forward step, that is, `slowing down training`.\n", - "\n", - "Summarizing: you will be training larger models (for example 7B in colab), but at the expense of training speed." - ], - "metadata": { - "id": "rYpCTXD1z48a" - } - }, - { - "cell_type": "code", - "source": [ - "# config = Config(\n", - "# model_name_or_path=\"facebook/opt-350m\",\n", - "\n", - "# use_gradient_checkpointing=True,\n", - "\n", - "# stabilize=True,\n", - "# apply_lora=True,\n", - "# load_in_4bit=True,\n", - "# prepare_model_for_kbit_training=True,\n", - "# )" - ], - "metadata": { - "id": "4-_uG-aN0s8F" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Add eval data" - ], - "metadata": { - "id": "UOG_A3itJzvR" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Setup config\n", - "\n", - "- `do_eval` for turn on evaluation \n", - "- `eval_steps` how often we should run evaluation" - ], - "metadata": { - "id": "Kw7FFI0TJ6hc" - } - }, - { - "cell_type": "code", - "source": [ - "config = Config(\n", - " model_name_or_path=\"facebook/opt-350m\",\n", - " stabilize=True,\n", - " apply_lora=True,\n", - " load_in_4bit=True,\n", - " prepare_model_for_kbit_training=True,\n", - " do_eval=True,\n", - " eval_steps=50,\n", - ")" - ], - "metadata": { - "id": "ue9wpOUy6--4" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Make dummy eval dataset" - ], - "metadata": { - "id": "xMIA8MwoKM8V" - } - }, - { - "cell_type": "code", - "source": [ - "eval_data = [\"Hi\", \"Sup?\"] * 10" - ], - "metadata": { - "id": "BLEMaWo2HJVs" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Make a `xllm` eval dataset" - ], - "metadata": { - "id": "EF2U5TZMKUDW" - } - }, - { - "cell_type": "code", - "source": [ - "eval_dataset = GeneralDataset.from_list(eval_data)" - ], - "metadata": { - "id": "exF7XB2AKRcn" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Init experiment with the `eval_dataset`" - ], - "metadata": { - "id": "oyHOGsJuQg0D" - } - }, - { - "cell_type": "code", - "source": [ - "experiment = Experiment(config=config, train_dataset=train_dataset, eval_dataset=eval_dataset)" - ], - "metadata": { - "id": "ps856ZWwKdYg" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Build experiment" - ], - "metadata": { - "id": "DC81sYGfQkDy" - } + "output_type": "stream", + "name": "stderr", + "text": [ + "Saving model checkpoint to ./outputs/checkpoint-100\n", + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "\u001b[32m2023-11-14 16:01:09.884\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:09.888\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## You also can add `Gradient Checkpointing`\n", + "\n", + "This will help to use `less GPU memory` during training, that is, you will be able to learn more than without this technique. The disadvantages of this technique is slowing down the forward step, that is, `slowing down training`.\n", + "\n", + "Summarizing: you will be training larger models (for example 7B in colab), but at the expense of training speed." + ], + "metadata": { + "id": "rYpCTXD1z48a" + } + }, + { + "cell_type": "code", + "source": [ + "# config = Config(\n", + "# model_name_or_path=\"facebook/opt-350m\",\n", + "\n", + "# use_gradient_checkpointing=True,\n", + "\n", + "# stabilize=True,\n", + "# apply_lora=True,\n", + "# load_in_4bit=True,\n", + "# prepare_model_for_kbit_training=True,\n", + "# )" + ], + "metadata": { + "id": "4-_uG-aN0s8F" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Add eval data" + ], + "metadata": { + "id": "UOG_A3itJzvR" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Setup config\n", + "\n", + "- `do_eval` for turn on evaluation \n", + "- `eval_steps` how often we should run evaluation" + ], + "metadata": { + "id": "Kw7FFI0TJ6hc" + } + }, + { + "cell_type": "code", + "source": [ + "config = Config(\n", + " model_name_or_path=\"facebook/opt-350m\",\n", + " stabilize=True,\n", + " apply_lora=True,\n", + " load_in_4bit=True,\n", + " prepare_model_for_kbit_training=True,\n", + " do_eval=True,\n", + " eval_steps=50,\n", + ")" + ], + "metadata": { + "id": "ue9wpOUy6--4" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Make dummy eval dataset" + ], + "metadata": { + "id": "xMIA8MwoKM8V" + } + }, + { + "cell_type": "code", + "source": [ + "eval_data = [\"Hi\", \"Sup?\"] * 10" + ], + "metadata": { + "id": "BLEMaWo2HJVs" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Make a `xllm` eval dataset" + ], + "metadata": { + "id": "EF2U5TZMKUDW" + } + }, + { + "cell_type": "code", + "source": [ + "eval_dataset = GeneralDataset.from_list(eval_data)" + ], + "metadata": { + "id": "exF7XB2AKRcn" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Init experiment with the `eval_dataset`" + ], + "metadata": { + "id": "oyHOGsJuQg0D" + } + }, + { + "cell_type": "code", + "source": [ + "experiment = Experiment(config=config, train_dataset=train_dataset, eval_dataset=eval_dataset)" + ], + "metadata": { + "id": "ps856ZWwKdYg" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Build experiment" + ], + "metadata": { + "id": "DC81sYGfQkDy" + } + }, + { + "cell_type": "code", + "source": [ + "experiment.build()" + ], + "metadata": { + "id": "k5LKSvT3KaYS", + "colab": { + "base_uri": "https://localhost:8080/" }, + "outputId": "29a263e6-28a4-4774-90bb-f3752af59ce5" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "code", - "source": [ - "experiment.build()" - ], - "metadata": { - "id": "k5LKSvT3KaYS", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "29a263e6-28a4-4774-90bb-f3752af59ce5" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 16:01:09.943\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:09.947\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", - "{\n", - " \"experiment_key\": \"base\",\n", - " \"save_safetensors\": true,\n", - " \"max_shard_size\": \"10GB\",\n", - " \"local_rank\": 0,\n", - " \"use_gradient_checkpointing\": false,\n", - " \"trainer_key\": \"lm\",\n", - " \"force_fp32\": false,\n", - " \"force_fp16\": false,\n", - " \"from_gptq\": false,\n", - " \"huggingface_hub_token\": null,\n", - " \"deepspeed_stage\": 0,\n", - " \"deepspeed_config_path\": null,\n", - " \"fsdp_strategy\": \"\",\n", - " \"fsdp_offload\": true,\n", - " \"seed\": 42,\n", - " \"stabilize\": true,\n", - " \"path_to_env_file\": \"./.env\",\n", - " \"prepare_dataset\": true,\n", - " \"lora_hub_model_id\": null,\n", - " \"lora_model_local_path\": null,\n", - " \"fused_model_local_path\": null,\n", - " \"fuse_after_training\": false,\n", - " \"quantization_dataset_id\": null,\n", - " \"quantization_max_samples\": 1024,\n", - " \"quantized_model_path\": \"./quantized_model/\",\n", - " \"quantized_hub_model_id\": null,\n", - " \"quantized_hub_private_repo\": true,\n", - " \"dataset_key\": \"soda\",\n", - " \"train_local_path_to_data\": \"./train.jsonl\",\n", - " \"eval_local_path_to_data\": null,\n", - " \"shuffle\": true,\n", - " \"max_eval_samples\": 1000,\n", - " \"add_eval_to_train_if_no_path\": false,\n", - " \"tokenizer_name_or_path\": null,\n", - " \"tokenizer_use_fast\": null,\n", - " \"tokenizer_padding_side\": null,\n", - " \"collator_key\": \"lm\",\n", - " \"max_length\": 2048,\n", - " \"model_name_or_path\": \"facebook/opt-350m\",\n", - " \"push_to_hub_bos_add_bos_token\": false,\n", - " \"use_flash_attention_2\": false,\n", - " \"trust_remote_code\": false,\n", - " \"device_map\": null,\n", - " \"prepare_model_for_kbit_training\": true,\n", - " \"load_in_8bit\": false,\n", - " \"load_in_4bit\": true,\n", - " \"llm_int8_threshold\": 6.0,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_quantize_after_model_init\": false,\n", - " \"gptq_bits\": 4,\n", - " \"gptq_group_size\": 128,\n", - " \"gptq_disable_exllama\": true,\n", - " \"apply_lora\": true,\n", - " \"lora_rank\": 8,\n", - " \"lora_alpha\": 32,\n", - " \"lora_dropout\": 0.1,\n", - " \"raw_lora_target_modules\": \"all\",\n", - " \"output_dir\": \"./outputs/\",\n", - " \"per_device_train_batch_size\": 2,\n", - " \"do_eval\": true,\n", - " \"per_device_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": null,\n", - " \"eval_delay\": 0,\n", - " \"eval_steps\": 50,\n", - " \"warmup_steps\": 1000,\n", - " \"max_steps\": null,\n", - " \"num_train_epochs\": 1,\n", - " \"learning_rate\": 0.0002,\n", - " \"max_grad_norm\": 1.0,\n", - " \"weight_decay\": 0.001,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"logging_steps\": 10,\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"push_to_hub\": false,\n", - " \"hub_model_id\": null,\n", - " \"hub_private_repo\": true,\n", - " \"report_to_wandb\": false,\n", - " \"wandb_api_key\": null,\n", - " \"wandb_project\": null,\n", - " \"wandb_entity\": null\n", - "}\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:09.949\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:09.950\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", - "PyTorch: setting up devices\n", - "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", - "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", - "\u001b[32m2023-11-14 16:01:09.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", - "{\n", - " \"output_dir\": \"./outputs/\",\n", - " \"overwrite_output_dir\": false,\n", - " \"do_train\": false,\n", - " \"do_eval\": true,\n", - " \"do_predict\": false,\n", - " \"evaluation_strategy\": \"steps\",\n", - " \"prediction_loss_only\": false,\n", - " \"per_device_train_batch_size\": 2,\n", - " \"per_device_eval_batch_size\": 2,\n", - " \"per_gpu_train_batch_size\": null,\n", - " \"per_gpu_eval_batch_size\": null,\n", - " \"gradient_accumulation_steps\": 1,\n", - " \"eval_accumulation_steps\": 1,\n", - " \"eval_delay\": 0,\n", - " \"learning_rate\": 0.0002,\n", - " \"weight_decay\": 0.001,\n", - " \"adam_beta1\": 0.9,\n", - " \"adam_beta2\": 0.999,\n", - " \"adam_epsilon\": 1e-08,\n", - " \"max_grad_norm\": 1.0,\n", - " \"num_train_epochs\": 1,\n", - " \"max_steps\": -1,\n", - " \"lr_scheduler_type\": \"linear\",\n", - " \"warmup_ratio\": 0.0,\n", - " \"warmup_steps\": 1000,\n", - " \"log_level\": \"info\",\n", - " \"log_level_replica\": \"warning\",\n", - " \"log_on_each_node\": true,\n", - " \"logging_dir\": \"./outputs/runs/Nov14_16-01-09_735f762378cc\",\n", - " \"logging_strategy\": \"steps\",\n", - " \"logging_first_step\": true,\n", - " \"logging_steps\": 10,\n", - " \"logging_nan_inf_filter\": true,\n", - " \"save_strategy\": \"steps\",\n", - " \"save_steps\": 100,\n", - " \"save_total_limit\": 1,\n", - " \"save_safetensors\": true,\n", - " \"save_on_each_node\": false,\n", - " \"no_cuda\": false,\n", - " \"use_cpu\": false,\n", - " \"use_mps_device\": false,\n", - " \"seed\": 42,\n", - " \"data_seed\": 42,\n", - " \"jit_mode_eval\": false,\n", - " \"use_ipex\": false,\n", - " \"bf16\": false,\n", - " \"fp16\": true,\n", - " \"fp16_opt_level\": \"O1\",\n", - " \"half_precision_backend\": \"auto\",\n", - " \"bf16_full_eval\": false,\n", - " \"fp16_full_eval\": false,\n", - " \"tf32\": null,\n", - " \"local_rank\": 0,\n", - " \"ddp_backend\": null,\n", - " \"tpu_num_cores\": null,\n", - " \"tpu_metrics_debug\": false,\n", - " \"debug\": [],\n", - " \"dataloader_drop_last\": false,\n", - " \"eval_steps\": 50,\n", - " \"dataloader_num_workers\": 0,\n", - " \"past_index\": -1,\n", - " \"run_name\": \"./outputs/\",\n", - " \"disable_tqdm\": false,\n", - " \"remove_unused_columns\": false,\n", - " \"label_names\": null,\n", - " \"load_best_model_at_end\": false,\n", - " \"metric_for_best_model\": \"eval_loss\",\n", - " \"greater_is_better\": false,\n", - " \"ignore_data_skip\": false,\n", - " \"fsdp\": [],\n", - " \"fsdp_min_num_params\": 0,\n", - " \"fsdp_config\": {\n", - " \"min_num_params\": 0,\n", - " \"xla\": false,\n", - " \"xla_fsdp_grad_ckpt\": false\n", - " },\n", - " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", - " \"deepspeed\": null,\n", - " \"label_smoothing_factor\": 0.0,\n", - " \"optim\": \"paged_adamw_8bit\",\n", - " \"optim_args\": null,\n", - " \"adafactor\": false,\n", - " \"group_by_length\": false,\n", - " \"length_column_name\": \"length\",\n", - " \"report_to\": [\n", - " \"tensorboard\"\n", - " ],\n", - " \"ddp_find_unused_parameters\": null,\n", - " \"ddp_bucket_cap_mb\": null,\n", - " \"ddp_broadcast_buffers\": null,\n", - " \"dataloader_pin_memory\": true,\n", - " \"skip_memory_metrics\": true,\n", - " \"use_legacy_prediction_loop\": false,\n", - " \"push_to_hub\": false,\n", - " \"resume_from_checkpoint\": null,\n", - " \"hub_model_id\": null,\n", - " \"hub_strategy\": \"checkpoint\",\n", - " \"hub_token\": \"\",\n", - " \"hub_private_repo\": true,\n", - " \"hub_always_push\": false,\n", - " \"gradient_checkpointing\": false,\n", - " \"gradient_checkpointing_kwargs\": null,\n", - " \"include_inputs_for_metrics\": false,\n", - " \"fp16_backend\": \"auto\",\n", - " \"push_to_hub_model_id\": null,\n", - " \"push_to_hub_organization\": null,\n", - " \"push_to_hub_token\": \"\",\n", - " \"mp_parameters\": \"\",\n", - " \"auto_find_batch_size\": false,\n", - " \"full_determinism\": false,\n", - " \"torchdynamo\": null,\n", - " \"ray_scope\": \"last\",\n", - " \"ddp_timeout\": 1800,\n", - " \"torch_compile\": false,\n", - " \"torch_compile_backend\": null,\n", - " \"torch_compile_mode\": null,\n", - " \"dispatch_batches\": null,\n", - " \"split_batches\": false,\n", - " \"include_tokens_per_second\": false,\n", - " \"neftune_noise_alpha\": null\n", - "}\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", - "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", - "loading file tokenizer.json from cache at None\n", - "loading file added_tokens.json from cache at None\n", - "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", - "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 16:01:10.243\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:10.244\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:10.249\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config was built:\n", - "{\n", - " \"bnb_4bit_compute_dtype\": \"float16\",\n", - " \"bnb_4bit_quant_type\": \"nf4\",\n", - " \"bnb_4bit_use_double_quant\": true,\n", - " \"llm_int8_has_fp16_weight\": true,\n", - " \"load_in_4bit\": true\n", - "}\n", - "\u001b[0m\n", - "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", - "Model config OPTConfig {\n", - " \"_name_or_path\": \"facebook/opt-350m\",\n", - " \"_remove_final_layer_norm\": false,\n", - " \"activation_dropout\": 0.0,\n", - " \"activation_function\": \"relu\",\n", - " \"architectures\": [\n", - " \"OPTForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0.0,\n", - " \"bos_token_id\": 2,\n", - " \"do_layer_norm_before\": false,\n", - " \"dropout\": 0.1,\n", - " \"enable_bias\": true,\n", - " \"eos_token_id\": 2,\n", - " \"ffn_dim\": 4096,\n", - " \"hidden_size\": 1024,\n", - " \"init_std\": 0.02,\n", - " \"layer_norm_elementwise_affine\": true,\n", - " \"layerdrop\": 0.0,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"opt\",\n", - " \"num_attention_heads\": 16,\n", - " \"num_hidden_layers\": 24,\n", - " \"pad_token_id\": 1,\n", - " \"prefix\": \"\",\n", - " \"torch_dtype\": \"float16\",\n", - " \"transformers_version\": \"4.35.1\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50272,\n", - " \"word_embed_proj_dim\": 512\n", - "}\n", - "\n", - "The device_map was not initialized. Setting device_map to {'':torch.cuda.current_device()}. If you want to use the model for inference, please set device_map ='auto' \n", - "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", - "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "Detected 4-bit loading: activating 4-bit loading for this model\n", - "All model checkpoint weights were used when initializing OPTForCausalLM.\n", - "\n", - "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", - "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", - "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", - "Generate config GenerationConfig {\n", - " \"bos_token_id\": 2,\n", - " \"eos_token_id\": 2,\n", - " \"pad_token_id\": 1\n", - "}\n", - "\n", - "\u001b[32m2023-11-14 16:01:12.087\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel prepared for kbit training. Gradient checkpointing: False\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:12.089\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:12.249\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:12.258\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", - "Using auto half precision backend\n", - "\u001b[32m2023-11-14 16:01:12.265\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", - "\u001b[32m2023-11-14 16:01:12.267\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n" - ] - } - ] + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 16:01:09.943\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment building has started\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:09.947\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig:\n", + "{\n", + " \"experiment_key\": \"base\",\n", + " \"save_safetensors\": true,\n", + " \"max_shard_size\": \"10GB\",\n", + " \"local_rank\": 0,\n", + " \"use_gradient_checkpointing\": false,\n", + " \"trainer_key\": \"lm\",\n", + " \"force_fp32\": false,\n", + " \"force_fp16\": false,\n", + " \"from_gptq\": false,\n", + " \"huggingface_hub_token\": null,\n", + " \"deepspeed_stage\": 0,\n", + " \"deepspeed_config_path\": null,\n", + " \"fsdp_strategy\": \"\",\n", + " \"fsdp_offload\": true,\n", + " \"seed\": 42,\n", + " \"stabilize\": true,\n", + " \"path_to_env_file\": \"./.env\",\n", + " \"prepare_dataset\": true,\n", + " \"lora_hub_model_id\": null,\n", + " \"lora_model_local_path\": null,\n", + " \"fused_model_local_path\": null,\n", + " \"fuse_after_training\": false,\n", + " \"quantization_dataset_id\": null,\n", + " \"quantization_max_samples\": 1024,\n", + " \"quantized_model_path\": \"./quantized_model/\",\n", + " \"quantized_hub_model_id\": null,\n", + " \"quantized_hub_private_repo\": true,\n", + " \"dataset_key\": \"soda\",\n", + " \"train_local_path_to_data\": \"./train.jsonl\",\n", + " \"eval_local_path_to_data\": null,\n", + " \"shuffle\": true,\n", + " \"max_eval_samples\": 1000,\n", + " \"add_eval_to_train_if_no_path\": false,\n", + " \"tokenizer_name_or_path\": null,\n", + " \"tokenizer_use_fast\": null,\n", + " \"tokenizer_padding_side\": null,\n", + " \"collator_key\": \"lm\",\n", + " \"max_length\": 2048,\n", + " \"model_name_or_path\": \"facebook/opt-350m\",\n", + " \"push_to_hub_bos_add_bos_token\": false,\n", + " \"use_flash_attention_2\": false,\n", + " \"trust_remote_code\": false,\n", + " \"device_map\": null,\n", + " \"prepare_model_for_kbit_training\": true,\n", + " \"load_in_8bit\": false,\n", + " \"load_in_4bit\": true,\n", + " \"llm_int8_threshold\": 6.0,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_quantize_after_model_init\": false,\n", + " \"gptq_bits\": 4,\n", + " \"gptq_group_size\": 128,\n", + " \"gptq_disable_exllama\": true,\n", + " \"apply_lora\": true,\n", + " \"lora_rank\": 8,\n", + " \"lora_alpha\": 32,\n", + " \"lora_dropout\": 0.1,\n", + " \"raw_lora_target_modules\": \"all\",\n", + " \"output_dir\": \"./outputs/\",\n", + " \"per_device_train_batch_size\": 2,\n", + " \"do_eval\": true,\n", + " \"per_device_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": null,\n", + " \"eval_delay\": 0,\n", + " \"eval_steps\": 50,\n", + " \"warmup_steps\": 1000,\n", + " \"max_steps\": null,\n", + " \"num_train_epochs\": 1,\n", + " \"learning_rate\": 0.0002,\n", + " \"max_grad_norm\": 1.0,\n", + " \"weight_decay\": 0.001,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"logging_steps\": 10,\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"push_to_hub\": false,\n", + " \"hub_model_id\": null,\n", + " \"hub_private_repo\": true,\n", + " \"report_to_wandb\": false,\n", + " \"wandb_api_key\": null,\n", + " \"wandb_project\": null,\n", + " \"wandb_entity\": null\n", + "}\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:09.949\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mConfig saved\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:09.950\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mChecks passed successfully\u001b[0m\n", + "PyTorch: setting up devices\n", + "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", + "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n", + "\u001b[32m2023-11-14 16:01:09.957\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining arguments was built:\n", + "{\n", + " \"output_dir\": \"./outputs/\",\n", + " \"overwrite_output_dir\": false,\n", + " \"do_train\": false,\n", + " \"do_eval\": true,\n", + " \"do_predict\": false,\n", + " \"evaluation_strategy\": \"steps\",\n", + " \"prediction_loss_only\": false,\n", + " \"per_device_train_batch_size\": 2,\n", + " \"per_device_eval_batch_size\": 2,\n", + " \"per_gpu_train_batch_size\": null,\n", + " \"per_gpu_eval_batch_size\": null,\n", + " \"gradient_accumulation_steps\": 1,\n", + " \"eval_accumulation_steps\": 1,\n", + " \"eval_delay\": 0,\n", + " \"learning_rate\": 0.0002,\n", + " \"weight_decay\": 0.001,\n", + " \"adam_beta1\": 0.9,\n", + " \"adam_beta2\": 0.999,\n", + " \"adam_epsilon\": 1e-08,\n", + " \"max_grad_norm\": 1.0,\n", + " \"num_train_epochs\": 1,\n", + " \"max_steps\": -1,\n", + " \"lr_scheduler_type\": \"linear\",\n", + " \"warmup_ratio\": 0.0,\n", + " \"warmup_steps\": 1000,\n", + " \"log_level\": \"info\",\n", + " \"log_level_replica\": \"warning\",\n", + " \"log_on_each_node\": true,\n", + " \"logging_dir\": \"./outputs/runs/Nov14_16-01-09_735f762378cc\",\n", + " \"logging_strategy\": \"steps\",\n", + " \"logging_first_step\": true,\n", + " \"logging_steps\": 10,\n", + " \"logging_nan_inf_filter\": true,\n", + " \"save_strategy\": \"steps\",\n", + " \"save_steps\": 100,\n", + " \"save_total_limit\": 1,\n", + " \"save_safetensors\": true,\n", + " \"save_on_each_node\": false,\n", + " \"no_cuda\": false,\n", + " \"use_cpu\": false,\n", + " \"use_mps_device\": false,\n", + " \"seed\": 42,\n", + " \"data_seed\": 42,\n", + " \"jit_mode_eval\": false,\n", + " \"use_ipex\": false,\n", + " \"bf16\": false,\n", + " \"fp16\": true,\n", + " \"fp16_opt_level\": \"O1\",\n", + " \"half_precision_backend\": \"auto\",\n", + " \"bf16_full_eval\": false,\n", + " \"fp16_full_eval\": false,\n", + " \"tf32\": null,\n", + " \"local_rank\": 0,\n", + " \"ddp_backend\": null,\n", + " \"tpu_num_cores\": null,\n", + " \"tpu_metrics_debug\": false,\n", + " \"debug\": [],\n", + " \"dataloader_drop_last\": false,\n", + " \"eval_steps\": 50,\n", + " \"dataloader_num_workers\": 0,\n", + " \"past_index\": -1,\n", + " \"run_name\": \"./outputs/\",\n", + " \"disable_tqdm\": false,\n", + " \"remove_unused_columns\": false,\n", + " \"label_names\": null,\n", + " \"load_best_model_at_end\": false,\n", + " \"metric_for_best_model\": \"eval_loss\",\n", + " \"greater_is_better\": false,\n", + " \"ignore_data_skip\": false,\n", + " \"fsdp\": [],\n", + " \"fsdp_min_num_params\": 0,\n", + " \"fsdp_config\": {\n", + " \"min_num_params\": 0,\n", + " \"xla\": false,\n", + " \"xla_fsdp_grad_ckpt\": false\n", + " },\n", + " \"fsdp_transformer_layer_cls_to_wrap\": null,\n", + " \"deepspeed\": null,\n", + " \"label_smoothing_factor\": 0.0,\n", + " \"optim\": \"paged_adamw_8bit\",\n", + " \"optim_args\": null,\n", + " \"adafactor\": false,\n", + " \"group_by_length\": false,\n", + " \"length_column_name\": \"length\",\n", + " \"report_to\": [\n", + " \"tensorboard\"\n", + " ],\n", + " \"ddp_find_unused_parameters\": null,\n", + " \"ddp_bucket_cap_mb\": null,\n", + " \"ddp_broadcast_buffers\": null,\n", + " \"dataloader_pin_memory\": true,\n", + " \"skip_memory_metrics\": true,\n", + " \"use_legacy_prediction_loop\": false,\n", + " \"push_to_hub\": false,\n", + " \"resume_from_checkpoint\": null,\n", + " \"hub_model_id\": null,\n", + " \"hub_strategy\": \"checkpoint\",\n", + " \"hub_token\": \"\",\n", + " \"hub_private_repo\": true,\n", + " \"hub_always_push\": false,\n", + " \"gradient_checkpointing\": false,\n", + " \"gradient_checkpointing_kwargs\": null,\n", + " \"include_inputs_for_metrics\": false,\n", + " \"fp16_backend\": \"auto\",\n", + " \"push_to_hub_model_id\": null,\n", + " \"push_to_hub_organization\": null,\n", + " \"push_to_hub_token\": \"\",\n", + " \"mp_parameters\": \"\",\n", + " \"auto_find_batch_size\": false,\n", + " \"full_determinism\": false,\n", + " \"torchdynamo\": null,\n", + " \"ray_scope\": \"last\",\n", + " \"ddp_timeout\": 1800,\n", + " \"torch_compile\": false,\n", + " \"torch_compile_backend\": null,\n", + " \"torch_compile_mode\": null,\n", + " \"dispatch_batches\": null,\n", + " \"split_batches\": false,\n", + " \"include_tokens_per_second\": false,\n", + " \"neftune_noise_alpha\": null\n", + "}\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/vocab.json\n", + "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/merges.txt\n", + "loading file tokenizer.json from cache at None\n", + "loading file added_tokens.json from cache at None\n", + "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/special_tokens_map.json\n", + "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/tokenizer_config.json\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 16:01:10.243\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTokenizer facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:10.244\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mCollator LMCollator was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:10.249\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mQuantization config was built:\n", + "{\n", + " \"bnb_4bit_compute_dtype\": \"float16\",\n", + " \"bnb_4bit_quant_type\": \"nf4\",\n", + " \"bnb_4bit_use_double_quant\": true,\n", + " \"llm_int8_has_fp16_weight\": true,\n", + " \"load_in_4bit\": true\n", + "}\n", + "\u001b[0m\n", + "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/config.json\n", + "Model config OPTConfig {\n", + " \"_name_or_path\": \"facebook/opt-350m\",\n", + " \"_remove_final_layer_norm\": false,\n", + " \"activation_dropout\": 0.0,\n", + " \"activation_function\": \"relu\",\n", + " \"architectures\": [\n", + " \"OPTForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 2,\n", + " \"do_layer_norm_before\": false,\n", + " \"dropout\": 0.1,\n", + " \"enable_bias\": true,\n", + " \"eos_token_id\": 2,\n", + " \"ffn_dim\": 4096,\n", + " \"hidden_size\": 1024,\n", + " \"init_std\": 0.02,\n", + " \"layer_norm_elementwise_affine\": true,\n", + " \"layerdrop\": 0.0,\n", + " \"max_position_embeddings\": 2048,\n", + " \"model_type\": \"opt\",\n", + " \"num_attention_heads\": 16,\n", + " \"num_hidden_layers\": 24,\n", + " \"pad_token_id\": 1,\n", + " \"prefix\": \"\",\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.35.1\",\n", + " \"use_cache\": true,\n", + " \"vocab_size\": 50272,\n", + " \"word_embed_proj_dim\": 512\n", + "}\n", + "\n", + "The device_map was not initialized. Setting device_map to {'':torch.cuda.current_device()}. If you want to use the model for inference, please set device_map ='auto' \n", + "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/pytorch_model.bin\n", + "Instantiating OPTForCausalLM model under default dtype torch.float16.\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "Detected 4-bit loading: activating 4-bit loading for this model\n", + "All model checkpoint weights were used when initializing OPTForCausalLM.\n", + "\n", + "All the weights of OPTForCausalLM were initialized from the model checkpoint at facebook/opt-350m.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use OPTForCausalLM for predictions without further training.\n", + "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--facebook--opt-350m/snapshots/08ab08cc4b72ff5593870b5d527cf4230323703c/generation_config.json\n", + "Generate config GenerationConfig {\n", + " \"bos_token_id\": 2,\n", + " \"eos_token_id\": 2,\n", + " \"pad_token_id\": 1\n", + "}\n", + "\n", + "\u001b[32m2023-11-14 16:01:12.087\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel prepared for kbit training. Gradient checkpointing: False\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:12.089\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:12.249\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mLoRA applied to the model facebook/opt-350m\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:12.258\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel facebook/opt-350m is stabilized for training\u001b[0m\n", + "Using auto half precision backend\n", + "\u001b[32m2023-11-14 16:01:12.265\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTrainer LMTrainer was built\u001b[0m\n", + "\u001b[32m2023-11-14 16:01:12.267\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mExperiment built successfully\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Run experiment" + ], + "metadata": { + "id": "jRCjV7UEQlhp" + } + }, + { + "cell_type": "code", + "source": [ + "experiment.run()" + ], + "metadata": { + "id": "NadhNd39KRRq", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 634 }, + "outputId": "93ad2638-92fe-4e40-9e45-1db33e0f82e5" + }, + "execution_count": null, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "## Run experiment" - ], - "metadata": { - "id": "jRCjV7UEQlhp" - } + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[32m2023-11-14 16:01:12.278\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", + "***** Running training *****\n", + " Num examples = 300\n", + " Num Epochs = 1\n", + " Instantaneous batch size per device = 2\n", + " Total train batch size (w. parallel, distributed & accumulation) = 2\n", + " Gradient Accumulation steps = 1\n", + " Total optimization steps = 150\n", + " Number of trainable parameters = 3,563,520\n" + ] }, { - "cell_type": "code", - "source": [ - "experiment.run()" + "output_type": "display_data", + "data": { + "text/plain": [ + "" ], - "metadata": { - "id": "NadhNd39KRRq", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 634 - }, - "outputId": "93ad2638-92fe-4e40-9e45-1db33e0f82e5" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\u001b[32m2023-11-14 16:01:12.278\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining will start soon\u001b[0m\n", - "***** Running training *****\n", - " Num examples = 300\n", - " Num Epochs = 1\n", - " Instantaneous batch size per device = 2\n", - " Total train batch size (w. parallel, distributed & accumulation) = 2\n", - " Gradient Accumulation steps = 1\n", - " Total optimization steps = 150\n", - " Number of trainable parameters = 3,563,520\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "

\n", - " \n", - " \n", - " [150/150 00:56, Epoch 1/1]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining LossValidation Loss
503.2167007.310188
1000.8291008.282642
1500.4285008.972834

" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "***** Running Evaluation *****\n", - " Num examples = 20\n", - " Batch size = 2\n", - "***** Running Evaluation *****\n", - " Num examples = 20\n", - " Batch size = 2\n", - "Saving model checkpoint to ./outputs/checkpoint-100\n", - "***** Running Evaluation *****\n", - " Num examples = 20\n", - " Batch size = 2\n", - "\n", - "\n", - "Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\n", - "\n", - "\u001b[32m2023-11-14 16:02:09.046\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", - "\u001b[32m2023-11-14 16:02:09.048\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" - ] - } + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [150/150 00:56, Epoch 1/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining LossValidation Loss
503.2167007.310188
1000.8291008.282642
1500.4285008.972834

" ] + }, + "metadata": {} }, { - "cell_type": "markdown", - "source": [ - "# 🎉 You are awesome!\n", - "\n", - "## Now you know how to prototype models using `xllm`\n", - "\n", - "### Explore more examples at X—LLM repo\n", - "\n", - "https://github.com/BobaZooba/xllm\n", - "\n", - "Useful materials:\n", - "\n", - "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", - "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", - "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", - "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", - " do\n", - "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", - " into your own project\n", - "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", - "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model\n" - ], - "metadata": { - "id": "NlX7tO65hOQU" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Tale Quest\n", - "\n", - "`Tale Quest` is my personal project which was built using `xllm` and `Shurale`. It's an interactive text-based game\n", - "in `Telegram` with dynamic AI characters, offering infinite scenarios\n", - "\n", - "You will get into exciting journeys and complete fascinating quests. Chat\n", - "with `George Orwell`, `Tech Entrepreneur`, `Young Wizard`, `Noir Detective`, `Femme Fatale` and many more\n", - "\n", - "Try it now: [https://t.me/talequestbot](https://t.me/TaleQuestBot?start=Z2g)" - ], - "metadata": { - "id": "5wJJrKnglAkK" - } - }, - { - "cell_type": "code", - "source": [], - "metadata": { - "id": "fFfDfFiCpDPv" - }, - "execution_count": null, - "outputs": [] + "output_type": "stream", + "name": "stderr", + "text": [ + "***** Running Evaluation *****\n", + " Num examples = 20\n", + " Batch size = 2\n", + "***** Running Evaluation *****\n", + " Num examples = 20\n", + " Batch size = 2\n", + "Saving model checkpoint to ./outputs/checkpoint-100\n", + "***** Running Evaluation *****\n", + " Num examples = 20\n", + " Batch size = 2\n", + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "\u001b[32m2023-11-14 16:02:09.046\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mTraining end\u001b[0m\n", + "\u001b[32m2023-11-14 16:02:09.048\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mxllm.utils.logger\u001b[0m:\u001b[36minfo\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mModel saved to ./outputs/\u001b[0m\n" + ] } - ] + ] + }, + { + "cell_type": "markdown", + "source": [ + "# 🎉 You are awesome!\n", + "\n", + "## Now you know how to prototype models using `xllm`\n", + "\n", + "### Explore more examples at X—LLM repo\n", + "\n", + "https://github.com/BobaZooba/xllm\n", + "\n", + "Useful materials:\n", + "\n", + "- [X—LLM Repo](https://github.com/BobaZooba/xllm): main repo of the `xllm` library\n", + "- [Quickstart](https://github.com/KompleteAI/xllm/tree/docs-v1#quickstart-): basics of `xllm`\n", + "- [Examples](https://github.com/BobaZooba/xllm/examples): minimal examples of using `xllm`\n", + "- [Guide](https://github.com/BobaZooba/xllm/blob/main/GUIDE.md): here, we go into detail about everything the library can\n", + " do\n", + "- [Demo project](https://github.com/BobaZooba/xllm-demo): here's a minimal step-by-step example of how to use X—LLM and fit it\n", + " into your own project\n", + "- [WeatherGPT](https://github.com/BobaZooba/wgpt): this repository features an example of how to utilize the xllm library. Included is a solution for a common type of assessment given to LLM engineers, who typically earn between $120,000 to $140,000 annually\n", + "- [Shurale](https://github.com/BobaZooba/shurale): project with the finetuned 7B Mistal model\n" + ], + "metadata": { + "id": "NlX7tO65hOQU" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Tale Quest\n", + "\n", + "`Tale Quest` is my personal project which was built using `xllm` and `Shurale`. It's an interactive text-based game\n", + "in `Telegram` with dynamic AI characters, offering infinite scenarios\n", + "\n", + "You will get into exciting journeys and complete fascinating quests. Chat\n", + "with `George Orwell`, `Tech Entrepreneur`, `Young Wizard`, `Noir Detective`, `Femme Fatale` and many more\n", + "\n", + "Try it now: [https://t.me/talequestbot](https://t.me/TaleQuestBot?start=Z2g)" + ], + "metadata": { + "id": "5wJJrKnglAkK" + } + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "fFfDfFiCpDPv" + }, + "execution_count": null, + "outputs": [] + } + ] } diff --git a/setup.py b/setup.py index a0cd640..f67266b 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ # Setup setup( name="xllm", - version="0.0.10", + version="0.1.0", description="Simple & Cutting Edge LLM Finetuning", license_files=["LICENSE"], long_description=open("README.md", "r", encoding="utf-8").read(), diff --git a/src/xllm/__init__.py b/src/xllm/__init__.py index 0539ccf..0cf6f71 100644 --- a/src/xllm/__init__.py +++ b/src/xllm/__init__.py @@ -14,7 +14,7 @@ # ruff: noqa: F401 -__version__ = "0.0.10" +__version__ = "0.1.0" from . import enums, types from .cli.fuse import cli_run_fuse diff --git a/src/xllm/core/config.py b/src/xllm/core/config.py index 46594ab..20a7147 100644 --- a/src/xllm/core/config.py +++ b/src/xllm/core/config.py @@ -344,8 +344,8 @@ class Config: "help": "Device map for loading the model", }, ) - prepare_model_for_kbit_training: bool = field( - default=True, + prepare_model_for_kbit_training: Optional[bool] = field( + default=None, metadata={ "help": "Prepare or not for kbit training", }, @@ -1069,3 +1069,10 @@ def lora_model_name_or_path_for_fusing(self) -> str: return self.lora_model_local_path else: raise ValueError("Please set lora_hub_model_id or lora_model_local_path for fusing") + + @property + def need_to_prepare_model_for_kbit_training(self) -> bool: + if self.prepare_model_for_kbit_training is not None: + return self.prepare_model_for_kbit_training + else: + return self.from_gptq or self.load_in_4bit or self.load_in_8bit diff --git a/src/xllm/core/dependencies.py b/src/xllm/core/dependencies.py index 43f9197..63cb337 100644 --- a/src/xllm/core/dependencies.py +++ b/src/xllm/core/dependencies.py @@ -453,7 +453,7 @@ def build_model( ) model.config.pretraining_tp = 1 - if quantization_config is not None and config.prepare_model_for_kbit_training: + if quantization_config is not None and config.need_to_prepare_model_for_kbit_training: model = prepare_model_for_kbit_training( model=model, use_gradient_checkpointing=config.use_gradient_checkpointing ) diff --git a/src/xllm/experiments/base.py b/src/xllm/experiments/base.py index 656871b..efb965f 100644 --- a/src/xllm/experiments/base.py +++ b/src/xllm/experiments/base.py @@ -456,7 +456,7 @@ def bnb_quantization(self) -> None: ) self.model.is_loaded_in_4bit = self.config.load_in_4bit self.model.is_loaded_in_8bit = self.config.load_in_8bit - if self.config.prepare_model_for_kbit_training: + if self.config.need_to_prepare_model_for_kbit_training: self.model = prepare_model_for_kbit_training( model=self.model, use_gradient_checkpointing=self.config.use_gradient_checkpointing )