Baby_Llama_4K.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OnipH4BHgpu1",
        "outputId": "3aebde4a-50de-4774-f7e0-42bf29a787cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'llama2.c'...\n",
            "remote: Enumerating objects: 1434, done.\u001b[K\n",
            "remote: Total 1434 (delta 0), reused 0 (delta 0), pack-reused 1434\u001b[K\n",
            "Receiving objects: 100% (1434/1434), 1.17 MiB | 12.21 MiB/s, done.\n",
            "Resolving deltas: 100% (871/871), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/karpathy/llama2.c.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install sentencepiece"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zRHKP8PpgxtI",
        "outputId": "892b8723-55ff-4c24-8396-08089460ab7a"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.99\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# runtime 100s\n",
        "!cd llama2.c && python tinystories.py download"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mWzFVICBv1sY",
        "outputId": "8a07f613-0621-4d9d-8934-692d9b664d88"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories_all_data.tar.gz to data/TinyStories_all_data.tar.gz...\n",
            "data/TinyStories_all_data.tar.gz: 100% 1.50G/1.50G [00:21<00:00, 75.8MiB/s]\n",
            "Unpacking data/TinyStories_all_data.tar.gz...\n",
            "Download done.\n",
            "Number of shards: 50\n",
            "Example story:\n",
            "{'story': '\\n\\nLily and Ben are friends. They like to play in the park. One day, they see a big tree with a swing. Lily wants to try the swing. She runs to the tree and climbs on the swing.\\n\"Push me, Ben!\" she says. Ben pushes her gently. Lily feels happy. She swings higher and higher. She laughs and shouts.\\nBen watches Lily. He thinks she is cute. He wants to swing too. He waits for Lily to stop. But Lily does not stop. She swings faster and faster. She is having too much fun.\\n\"Can I swing too, Lily?\" Ben asks. Lily does not hear him. She is too busy swinging. Ben feels sad. He walks away.\\nLily swings so high that she loses her grip. She falls off the swing. She lands on the ground. She hurts her foot. She cries.\\n\"Ow, ow, ow!\" she says. She looks for Ben. She wants him to help her. But Ben is not there. He is gone.\\nLily feels sorry. She wishes she had shared the swing with Ben. She wishes he was there to hug her. She limps to the tree. She sees something hanging from a branch. It is Ben\\'s hat. He left it for her.\\nLily smiles. She thinks Ben is nice. She puts on his hat. She hopes he will come back. She wants to say sorry. She wants to be friends again.', 'instruction': {'prompt:': 'Write a short story (3-5 paragraphs) which only uses very simple words that a 3 year old child would understand. The story should use the verb \"hang\", the noun \"foot\" and the adjective \"cute\". The story has the following features: the story should contain at least one dialogue. Remember to only use simple words!\\n\\nPossible story:', 'words': ['hang', 'foot', 'cute'], 'features': ['Dialogue']}, 'summary': 'Lily and Ben play in the park and Lily gets too caught up in swinging, causing Ben to leave. Lily falls off the swing and hurts herself, but Ben leaves his hat for her as a kind gesture.', 'source': 'GPT-4'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !cd llama2.c && python tinystories.py train_vocab --vocab_size=4096\n",
        "\n",
        "!cd ./llama2.c/data && wget https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/tok4096.vocab\n",
        "!cd ./llama2.c/data && wget https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/tok4096.model\n",
        "\n",
        "# !cd llama2.c && python tinystories.py pretokenize --vocab_size=4096\n",
        "!cd ./llama2.c/data && mkdir tok4096\n",
        "!cd ./llama2.c/data/tok4096 && wget https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/TinyStories_tok4096.tar.gz\n",
        "!cd ./llama2.c/data/tok4096 && tar -xf TinyStories_tok4096.tar.gz"
      ],
      "metadata": {
        "id": "Tx04_T-pv1zf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17f1ca71-50d3-4b62-b52b-77b2b35eed12"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-12-12 11:46:54--  https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/tok4096.vocab\n",
            "Resolving huggingface.co (huggingface.co)... 65.8.178.118, 65.8.178.27, 65.8.178.93, ...\n",
            "Connecting to huggingface.co (huggingface.co)|65.8.178.118|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 53534 (52K) [text/plain]\n",
            "Saving to: ‘tok4096.vocab’\n",
            "\n",
            "tok4096.vocab       100%[===================>]  52.28K  --.-KB/s    in 0.03s   \n",
            "\n",
            "2023-12-12 11:46:54 (1.88 MB/s) - ‘tok4096.vocab’ saved [53534/53534]\n",
            "\n",
            "--2023-12-12 11:46:54--  https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/tok4096.model\n",
            "Resolving huggingface.co (huggingface.co)... 65.8.178.118, 65.8.178.27, 65.8.178.93, ...\n",
            "Connecting to huggingface.co (huggingface.co)|65.8.178.118|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs-us-1.huggingface.co/repos/65/c8/65c8615bfd6b6d94c8e62381ee6653578b439f71ff04b86aaefa9828dec8bb70/645406d6a15820e53c4a1ec73a93ee10331a79edaeeabc08f1f09dcc786a9002?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27tok4096.model%3B+filename%3D%22tok4096.model%22%3B&Expires=1702640815&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMjY0MDgxNX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzY1L2M4LzY1Yzg2MTViZmQ2YjZkOTRjOGU2MjM4MWVlNjY1MzU3OGI0MzlmNzFmZjA0Yjg2YWFlZmE5ODI4ZGVjOGJiNzAvNjQ1NDA2ZDZhMTU4MjBlNTNjNGExZWM3M2E5M2VlMTAzMzFhNzllZGFlZWFiYzA4ZjFmMDlkY2M3ODZhOTAwMj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=kwM%7EBcBD6LXU5z-w8eTUSoxIWY6m7RUYOeyx2jqjW2sv9SG5N-dc7eTQwRiqguKSVpduxTWxYQ0pzSIwFjnIiBJ%7EweYpnqwXNBsv-W4Dp8kFUTlBWTm%7E-JYWS-psL9cVtzbVB-ePy5gbL-B7z9uOgSmd6CS2ZDKE-0zqqmq7o2aZ-UlPAUbeUWSBiYbfH%7EmAi%7EYFIkBHMd9-OTHrJObWweh-OFn8LPHeY8%7EqTLFStMKzU8OJMXjGbZOpMdy8ayZDabMCUUCiC4SwuyLEhC8KSHqPo8NcV3NWbZWX1LAXhIvFItQQE-UV5fhsBzVv8pDi2nZkYvsSi7Nhl3cyzXg0XQ__&Key-Pair-Id=KCD77M1F0VK2B [following]\n",
            "--2023-12-12 11:46:55--  https://cdn-lfs-us-1.huggingface.co/repos/65/c8/65c8615bfd6b6d94c8e62381ee6653578b439f71ff04b86aaefa9828dec8bb70/645406d6a15820e53c4a1ec73a93ee10331a79edaeeabc08f1f09dcc786a9002?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27tok4096.model%3B+filename%3D%22tok4096.model%22%3B&Expires=1702640815&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMjY0MDgxNX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzY1L2M4LzY1Yzg2MTViZmQ2YjZkOTRjOGU2MjM4MWVlNjY1MzU3OGI0MzlmNzFmZjA0Yjg2YWFlZmE5ODI4ZGVjOGJiNzAvNjQ1NDA2ZDZhMTU4MjBlNTNjNGExZWM3M2E5M2VlMTAzMzFhNzllZGFlZWFiYzA4ZjFmMDlkY2M3ODZhOTAwMj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=kwM%7EBcBD6LXU5z-w8eTUSoxIWY6m7RUYOeyx2jqjW2sv9SG5N-dc7eTQwRiqguKSVpduxTWxYQ0pzSIwFjnIiBJ%7EweYpnqwXNBsv-W4Dp8kFUTlBWTm%7E-JYWS-psL9cVtzbVB-ePy5gbL-B7z9uOgSmd6CS2ZDKE-0zqqmq7o2aZ-UlPAUbeUWSBiYbfH%7EmAi%7EYFIkBHMd9-OTHrJObWweh-OFn8LPHeY8%7EqTLFStMKzU8OJMXjGbZOpMdy8ayZDabMCUUCiC4SwuyLEhC8KSHqPo8NcV3NWbZWX1LAXhIvFItQQE-UV5fhsBzVv8pDi2nZkYvsSi7Nhl3cyzXg0XQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "Resolving cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)... 13.249.98.125, 13.249.98.16, 13.249.98.99, ...\n",
            "Connecting to cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)|13.249.98.125|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 64482 (63K) [binary/octet-stream]\n",
            "Saving to: ‘tok4096.model’\n",
            "\n",
            "tok4096.model       100%[===================>]  62.97K  --.-KB/s    in 0.05s   \n",
            "\n",
            "2023-12-12 11:46:55 (1.33 MB/s) - ‘tok4096.model’ saved [64482/64482]\n",
            "\n",
            "--2023-12-12 11:46:55--  https://huggingface.co/datasets/enio/TinyStories/resolve/main/tok4096/TinyStories_tok4096.tar.gz\n",
            "Resolving huggingface.co (huggingface.co)... 65.8.178.118, 65.8.178.27, 65.8.178.93, ...\n",
            "Connecting to huggingface.co (huggingface.co)|65.8.178.118|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs-us-1.huggingface.co/repos/65/c8/65c8615bfd6b6d94c8e62381ee6653578b439f71ff04b86aaefa9828dec8bb70/6d719c730b2beef6ec7a0f5d028651b94a28f5937023adbce78c95cd3e50299e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories_tok4096.tar.gz%3B+filename%3D%22TinyStories_tok4096.tar.gz%22%3B&response-content-type=application%2Fgzip&Expires=1702640815&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMjY0MDgxNX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzY1L2M4LzY1Yzg2MTViZmQ2YjZkOTRjOGU2MjM4MWVlNjY1MzU3OGI0MzlmNzFmZjA0Yjg2YWFlZmE5ODI4ZGVjOGJiNzAvNmQ3MTljNzMwYjJiZWVmNmVjN2EwZjVkMDI4NjUxYjk0YTI4ZjU5MzcwMjNhZGJjZTc4Yzk1Y2QzZTUwMjk5ZT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPSoifV19&Signature=GsHZ7ETMoPfOcfxTlLcG4RZM5YU8cAXxc%7Eoc7z5t97oxsdek%7E0dAhCUR9HDZGr%7ExR-Eq4C5J-orPYQCIxyi73mqiZC2rPI8PZBPJoC3jHdzzpPe0vx4b9ayczIp3UilqdXA%7EvOkfrKn5XhmD9Rf42aVqrJHl3BjSAHMQ32hpfWYtiKxliJfhNjcZqR9VT%7EpdINvF%7E6qA3OUag7mlGJCZmtZhaZ%7EuI5idyfFNhc4F41tFAnw1D0s%7EsiGZgR32OjoQQ6gbNcW1-vmGlTjFyuQ51t63D5D9jZolMX8PC1oBMDXBvMySmPvxi4cJlzLiMqQIU6wRzj3iOOjAwuPcvn1Piw__&Key-Pair-Id=KCD77M1F0VK2B [following]\n",
            "--2023-12-12 11:46:55--  https://cdn-lfs-us-1.huggingface.co/repos/65/c8/65c8615bfd6b6d94c8e62381ee6653578b439f71ff04b86aaefa9828dec8bb70/6d719c730b2beef6ec7a0f5d028651b94a28f5937023adbce78c95cd3e50299e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories_tok4096.tar.gz%3B+filename%3D%22TinyStories_tok4096.tar.gz%22%3B&response-content-type=application%2Fgzip&Expires=1702640815&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMjY0MDgxNX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzY1L2M4LzY1Yzg2MTViZmQ2YjZkOTRjOGU2MjM4MWVlNjY1MzU3OGI0MzlmNzFmZjA0Yjg2YWFlZmE5ODI4ZGVjOGJiNzAvNmQ3MTljNzMwYjJiZWVmNmVjN2EwZjVkMDI4NjUxYjk0YTI4ZjU5MzcwMjNhZGJjZTc4Yzk1Y2QzZTUwMjk5ZT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPSoifV19&Signature=GsHZ7ETMoPfOcfxTlLcG4RZM5YU8cAXxc%7Eoc7z5t97oxsdek%7E0dAhCUR9HDZGr%7ExR-Eq4C5J-orPYQCIxyi73mqiZC2rPI8PZBPJoC3jHdzzpPe0vx4b9ayczIp3UilqdXA%7EvOkfrKn5XhmD9Rf42aVqrJHl3BjSAHMQ32hpfWYtiKxliJfhNjcZqR9VT%7EpdINvF%7E6qA3OUag7mlGJCZmtZhaZ%7EuI5idyfFNhc4F41tFAnw1D0s%7EsiGZgR32OjoQQ6gbNcW1-vmGlTjFyuQ51t63D5D9jZolMX8PC1oBMDXBvMySmPvxi4cJlzLiMqQIU6wRzj3iOOjAwuPcvn1Piw__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "Resolving cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)... 13.249.98.125, 13.249.98.16, 13.249.98.99, ...\n",
            "Connecting to cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)|13.249.98.125|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 943741969 (900M) [application/gzip]\n",
            "Saving to: ‘TinyStories_tok4096.tar.gz’\n",
            "\n",
            "TinyStories_tok4096 100%[===================>] 900.02M  77.6MB/s    in 12s     \n",
            "\n",
            "2023-12-12 11:47:07 (75.2 MB/s) - ‘TinyStories_tok4096.tar.gz’ saved [943741969/943741969]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd llama2.c && python tokenizer.py --tokenizer-model=data/tok4096.model"
      ],
      "metadata": {
        "id": "lSezLzpnM8Cl"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# change bfloat16 to float16 on line 72\n",
        "!sed -i \"72s/.*/dtype = 'float16'  # float32|bfloat16|float16/\" ./llama2.c/train.py"
      ],
      "metadata": {
        "id": "tDQu756djGVD"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# runtime ~ 2 mins\n",
        "!cd llama2.c && python train.py --vocab_source=custom --vocab_size=4096 --compile=False \\\n",
        "  --dim=128 --n_layers=5 --n_heads=8 --n_kv_heads=4 --batch_size=32 \\\n",
        "  --always_save_checkpoint=True --eval_interval=500 --max_iters=1001 --init_from='resume'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sdMK1MW8g4eM",
        "outputId": "97f3dbe0-a506-45bc-d3f8-a00b72fbd9da"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overriding: vocab_source = custom\n",
            "Overriding: vocab_size = 4096\n",
            "Overriding: compile = False\n",
            "Overriding: dim = 128\n",
            "Overriding: n_layers = 5\n",
            "Overriding: n_heads = 8\n",
            "Overriding: n_kv_heads = 4\n",
            "Overriding: batch_size = 32\n",
            "Overriding: always_save_checkpoint = True\n",
            "Overriding: eval_interval = 500\n",
            "Overriding: max_iters = 1001\n",
            "Overriding: init_from = resume\n",
            "tokens per iteration will be: 32,768\n",
            "breaks down as: 4 grad accum steps * 1 processes * 32 batch size * 256 max seq len\n",
            "Resuming training from out\n",
            "num decayed parameter tensors: 36, with 1,445,888 parameters\n",
            "num non-decayed parameter tensors: 11, with 1,408 parameters\n",
            "using fused AdamW: True\n",
            "Created a PretokDataset with rng seed 42\n",
            "100 | loss 7.6606 | lr 5.000000e-05 | 741.51ms | mfu -100.00%\n",
            "101 | loss 7.6589 | lr 5.050000e-05 | 148.91ms | mfu -100.00%\n",
            "102 | loss 7.6510 | lr 5.100000e-05 | 127.00ms | mfu -100.00%\n",
            "103 | loss 7.6378 | lr 5.150000e-05 | 125.73ms | mfu -100.00%\n",
            "104 | loss 7.6287 | lr 5.200000e-05 | 126.58ms | mfu -100.00%\n",
            "105 | loss 7.6260 | lr 5.250000e-05 | 127.23ms | mfu 0.88%\n",
            "106 | loss 7.6178 | lr 5.300000e-05 | 128.25ms | mfu 0.88%\n",
            "107 | loss 7.6007 | lr 5.350000e-05 | 126.21ms | mfu 0.88%\n",
            "108 | loss 7.6124 | lr 5.400000e-05 | 128.01ms | mfu 0.88%\n",
            "109 | loss 7.5837 | lr 5.450000e-05 | 129.37ms | mfu 0.88%\n",
            "110 | loss 7.5581 | lr 5.500000e-05 | 127.61ms | mfu 0.88%\n",
            "111 | loss 7.5664 | lr 5.550000e-05 | 127.09ms | mfu 0.88%\n",
            "112 | loss 7.5483 | lr 5.600000e-05 | 127.77ms | mfu 0.88%\n",
            "113 | loss 7.5304 | lr 5.650000e-05 | 126.42ms | mfu 0.88%\n",
            "114 | loss 7.5297 | lr 5.700000e-05 | 126.57ms | mfu 0.88%\n",
            "115 | loss 7.5259 | lr 5.750000e-05 | 126.46ms | mfu 0.88%\n",
            "116 | loss 7.4964 | lr 5.800000e-05 | 127.13ms | mfu 0.88%\n",
            "117 | loss 7.5022 | lr 5.850000e-05 | 131.48ms | mfu 0.88%\n",
            "118 | loss 7.4890 | lr 5.900000e-05 | 127.75ms | mfu 0.88%\n",
            "119 | loss 7.4719 | lr 5.950000e-05 | 129.04ms | mfu 0.88%\n",
            "120 | loss 7.4622 | lr 6.000000e-05 | 129.39ms | mfu 0.87%\n",
            "121 | loss 7.4665 | lr 6.050000e-05 | 125.97ms | mfu 0.88%\n",
            "122 | loss 7.4258 | lr 6.100000e-05 | 127.44ms | mfu 0.88%\n",
            "123 | loss 7.4282 | lr 6.150000e-05 | 125.64ms | mfu 0.88%\n",
            "124 | loss 7.3935 | lr 6.200000e-05 | 128.85ms | mfu 0.88%\n",
            "125 | loss 7.3759 | lr 6.250000e-05 | 128.43ms | mfu 0.88%\n",
            "126 | loss 7.3838 | lr 6.300000e-05 | 127.05ms | mfu 0.88%\n",
            "127 | loss 7.3638 | lr 6.350000e-05 | 126.35ms | mfu 0.88%\n",
            "128 | loss 7.3597 | lr 6.400000e-05 | 128.72ms | mfu 0.88%\n",
            "129 | loss 7.3452 | lr 6.450000e-05 | 127.08ms | mfu 0.88%\n",
            "130 | loss 7.3336 | lr 6.500000e-05 | 126.28ms | mfu 0.88%\n",
            "131 | loss 7.3699 | lr 6.550000e-05 | 126.43ms | mfu 0.88%\n",
            "132 | loss 7.3133 | lr 6.600000e-05 | 128.46ms | mfu 0.88%\n",
            "133 | loss 7.3098 | lr 6.650000e-05 | 126.96ms | mfu 0.88%\n",
            "134 | loss 7.3073 | lr 6.700000e-05 | 126.04ms | mfu 0.88%\n",
            "135 | loss 7.3042 | lr 6.750000e-05 | 127.14ms | mfu 0.88%\n",
            "136 | loss 7.2992 | lr 6.800000e-05 | 129.79ms | mfu 0.88%\n",
            "137 | loss 7.2571 | lr 6.850000e-05 | 130.11ms | mfu 0.88%\n",
            "138 | loss 7.2337 | lr 6.900000e-05 | 126.62ms | mfu 0.88%\n",
            "139 | loss 7.2437 | lr 6.950000e-05 | 127.24ms | mfu 0.88%\n",
            "140 | loss 7.2222 | lr 7.000000e-05 | 127.99ms | mfu 0.88%\n",
            "141 | loss 7.2389 | lr 7.050000e-05 | 128.81ms | mfu 0.88%\n",
            "142 | loss 7.2004 | lr 7.100000e-05 | 126.97ms | mfu 0.88%\n",
            "143 | loss 7.1856 | lr 7.150000e-05 | 127.14ms | mfu 0.88%\n",
            "144 | loss 7.1598 | lr 7.200000e-05 | 129.60ms | mfu 0.88%\n",
            "145 | loss 7.1596 | lr 7.250000e-05 | 146.53ms | mfu 0.86%\n",
            "146 | loss 7.1352 | lr 7.300000e-05 | 134.94ms | mfu 0.86%\n",
            "147 | loss 7.1558 | lr 7.350000e-05 | 142.65ms | mfu 0.85%\n",
            "148 | loss 7.1113 | lr 7.400000e-05 | 136.41ms | mfu 0.85%\n",
            "149 | loss 7.1162 | lr 7.450000e-05 | 140.21ms | mfu 0.84%\n",
            "150 | loss 7.0915 | lr 7.500000e-05 | 152.77ms | mfu 0.83%\n",
            "151 | loss 7.1069 | lr 7.550000e-05 | 144.98ms | mfu 0.83%\n",
            "152 | loss 7.0854 | lr 7.600000e-05 | 136.77ms | mfu 0.83%\n",
            "153 | loss 7.0646 | lr 7.650000e-05 | 137.66ms | mfu 0.82%\n",
            "154 | loss 7.0448 | lr 7.700000e-05 | 138.96ms | mfu 0.82%\n",
            "155 | loss 7.0571 | lr 7.750000e-05 | 132.94ms | mfu 0.82%\n",
            "156 | loss 7.0042 | lr 7.800000e-05 | 133.14ms | mfu 0.83%\n",
            "157 | loss 7.0108 | lr 7.850000e-05 | 137.19ms | mfu 0.83%\n",
            "158 | loss 7.0015 | lr 7.900000e-05 | 134.27ms | mfu 0.83%\n",
            "159 | loss 6.9694 | lr 7.950000e-05 | 138.26ms | mfu 0.82%\n",
            "160 | loss 7.0120 | lr 8.000000e-05 | 138.15ms | mfu 0.82%\n",
            "161 | loss 6.9864 | lr 8.050000e-05 | 136.02ms | mfu 0.82%\n",
            "162 | loss 6.9785 | lr 8.100000e-05 | 138.98ms | mfu 0.82%\n",
            "163 | loss 6.9149 | lr 8.150000e-05 | 138.50ms | mfu 0.82%\n",
            "164 | loss 6.9250 | lr 8.200000e-05 | 134.03ms | mfu 0.82%\n",
            "165 | loss 6.9355 | lr 8.250000e-05 | 137.99ms | mfu 0.82%\n",
            "166 | loss 6.8802 | lr 8.300000e-05 | 134.06ms | mfu 0.82%\n",
            "167 | loss 6.8877 | lr 8.350000e-05 | 137.78ms | mfu 0.82%\n",
            "168 | loss 6.8932 | lr 8.400000e-05 | 135.21ms | mfu 0.82%\n",
            "169 | loss 6.8574 | lr 8.450000e-05 | 138.05ms | mfu 0.82%\n",
            "170 | loss 6.8769 | lr 8.500000e-05 | 139.87ms | mfu 0.82%\n",
            "171 | loss 6.8382 | lr 8.550000e-05 | 145.98ms | mfu 0.81%\n",
            "172 | loss 6.8345 | lr 8.600000e-05 | 130.48ms | mfu 0.82%\n",
            "173 | loss 6.8422 | lr 8.650000e-05 | 127.01ms | mfu 0.82%\n",
            "174 | loss 6.8040 | lr 8.700000e-05 | 129.63ms | mfu 0.83%\n",
            "175 | loss 6.8083 | lr 8.750000e-05 | 127.15ms | mfu 0.83%\n",
            "176 | loss 6.7800 | lr 8.800000e-05 | 128.90ms | mfu 0.84%\n",
            "177 | loss 6.7961 | lr 8.850000e-05 | 128.58ms | mfu 0.84%\n",
            "178 | loss 6.7619 | lr 8.900000e-05 | 128.22ms | mfu 0.84%\n",
            "179 | loss 6.7888 | lr 8.950000e-05 | 127.94ms | mfu 0.85%\n",
            "180 | loss 6.7259 | lr 9.000000e-05 | 127.08ms | mfu 0.85%\n",
            "181 | loss 6.7218 | lr 9.050000e-05 | 126.58ms | mfu 0.85%\n",
            "182 | loss 6.7421 | lr 9.100000e-05 | 128.76ms | mfu 0.85%\n",
            "183 | loss 6.6899 | lr 9.150000e-05 | 127.01ms | mfu 0.86%\n",
            "184 | loss 6.7067 | lr 9.200000e-05 | 128.45ms | mfu 0.86%\n",
            "185 | loss 6.6745 | lr 9.250000e-05 | 127.73ms | mfu 0.86%\n",
            "186 | loss 6.6727 | lr 9.300000e-05 | 127.72ms | mfu 0.86%\n",
            "187 | loss 6.6605 | lr 9.350000e-05 | 130.28ms | mfu 0.86%\n",
            "188 | loss 6.6257 | lr 9.400000e-05 | 127.34ms | mfu 0.86%\n",
            "189 | loss 6.5961 | lr 9.450000e-05 | 128.39ms | mfu 0.86%\n",
            "190 | loss 6.6306 | lr 9.500000e-05 | 129.20ms | mfu 0.86%\n",
            "191 | loss 6.5813 | lr 9.550000e-05 | 127.38ms | mfu 0.87%\n",
            "192 | loss 6.6311 | lr 9.600000e-05 | 129.22ms | mfu 0.87%\n",
            "193 | loss 6.5993 | lr 9.650000e-05 | 128.00ms | mfu 0.87%\n",
            "194 | loss 6.5514 | lr 9.700000e-05 | 127.29ms | mfu 0.87%\n",
            "195 | loss 6.5690 | lr 9.750000e-05 | 130.61ms | mfu 0.87%\n",
            "196 | loss 6.5357 | lr 9.800000e-05 | 126.86ms | mfu 0.87%\n",
            "197 | loss 6.5735 | lr 9.850000e-05 | 127.54ms | mfu 0.87%\n",
            "198 | loss 6.5711 | lr 9.900000e-05 | 130.72ms | mfu 0.87%\n",
            "199 | loss 6.4945 | lr 9.950000e-05 | 128.97ms | mfu 0.87%\n",
            "200 | loss 6.5081 | lr 1.000000e-04 | 127.15ms | mfu 0.87%\n",
            "201 | loss 6.5273 | lr 1.005000e-04 | 128.53ms | mfu 0.87%\n",
            "202 | loss 6.4933 | lr 1.010000e-04 | 127.47ms | mfu 0.87%\n",
            "203 | loss 6.4694 | lr 1.015000e-04 | 128.82ms | mfu 0.87%\n",
            "204 | loss 6.4342 | lr 1.020000e-04 | 127.98ms | mfu 0.87%\n",
            "205 | loss 6.4076 | lr 1.025000e-04 | 126.84ms | mfu 0.87%\n",
            "206 | loss 6.4368 | lr 1.030000e-04 | 129.78ms | mfu 0.87%\n",
            "207 | loss 6.4129 | lr 1.035000e-04 | 127.85ms | mfu 0.87%\n",
            "208 | loss 6.3748 | lr 1.040000e-04 | 126.75ms | mfu 0.87%\n",
            "209 | loss 6.4112 | lr 1.045000e-04 | 128.04ms | mfu 0.87%\n",
            "210 | loss 6.4097 | lr 1.050000e-04 | 129.27ms | mfu 0.87%\n",
            "211 | loss 6.4172 | lr 1.055000e-04 | 129.29ms | mfu 0.87%\n",
            "212 | loss 6.3780 | lr 1.060000e-04 | 128.34ms | mfu 0.87%\n",
            "213 | loss 6.3622 | lr 1.065000e-04 | 127.47ms | mfu 0.87%\n",
            "214 | loss 6.3398 | lr 1.070000e-04 | 129.49ms | mfu 0.87%\n",
            "215 | loss 6.3589 | lr 1.075000e-04 | 128.10ms | mfu 0.87%\n",
            "216 | loss 6.3414 | lr 1.080000e-04 | 129.86ms | mfu 0.87%\n",
            "217 | loss 6.3008 | lr 1.085000e-04 | 129.44ms | mfu 0.87%\n",
            "218 | loss 6.3047 | lr 1.090000e-04 | 127.64ms | mfu 0.87%\n",
            "219 | loss 6.2853 | lr 1.095000e-04 | 128.75ms | mfu 0.87%\n",
            "220 | loss 6.2896 | lr 1.100000e-04 | 127.33ms | mfu 0.87%\n",
            "221 | loss 6.2750 | lr 1.105000e-04 | 128.35ms | mfu 0.87%\n",
            "222 | loss 6.2199 | lr 1.110000e-04 | 130.67ms | mfu 0.87%\n",
            "223 | loss 6.2594 | lr 1.115000e-04 | 127.97ms | mfu 0.87%\n",
            "224 | loss 6.2117 | lr 1.120000e-04 | 128.90ms | mfu 0.87%\n",
            "225 | loss 6.1803 | lr 1.125000e-04 | 127.69ms | mfu 0.87%\n",
            "226 | loss 6.1754 | lr 1.130000e-04 | 127.90ms | mfu 0.87%\n",
            "227 | loss 6.1446 | lr 1.135000e-04 | 130.21ms | mfu 0.87%\n",
            "228 | loss 6.1564 | lr 1.140000e-04 | 127.40ms | mfu 0.87%\n",
            "229 | loss 6.1371 | lr 1.145000e-04 | 132.20ms | mfu 0.87%\n",
            "230 | loss 6.1546 | lr 1.150000e-04 | 129.27ms | mfu 0.87%\n",
            "231 | loss 6.1522 | lr 1.155000e-04 | 127.56ms | mfu 0.87%\n",
            "232 | loss 6.1274 | lr 1.160000e-04 | 128.95ms | mfu 0.87%\n",
            "233 | loss 6.1218 | lr 1.165000e-04 | 127.45ms | mfu 0.87%\n",
            "234 | loss 6.0931 | lr 1.170000e-04 | 128.91ms | mfu 0.87%\n",
            "235 | loss 6.0715 | lr 1.175000e-04 | 130.68ms | mfu 0.87%\n",
            "236 | loss 6.0869 | lr 1.180000e-04 | 127.40ms | mfu 0.87%\n",
            "237 | loss 6.1147 | lr 1.185000e-04 | 129.25ms | mfu 0.87%\n",
            "238 | loss 6.0958 | lr 1.190000e-04 | 128.17ms | mfu 0.87%\n",
            "239 | loss 6.0600 | lr 1.195000e-04 | 128.88ms | mfu 0.87%\n",
            "240 | loss 6.0938 | lr 1.200000e-04 | 128.89ms | mfu 0.87%\n",
            "241 | loss 6.0173 | lr 1.205000e-04 | 129.29ms | mfu 0.87%\n",
            "242 | loss 6.0330 | lr 1.210000e-04 | 131.41ms | mfu 0.87%\n",
            "243 | loss 6.0146 | lr 1.215000e-04 | 128.15ms | mfu 0.87%\n",
            "244 | loss 5.9823 | lr 1.220000e-04 | 128.65ms | mfu 0.87%\n",
            "245 | loss 5.9716 | lr 1.225000e-04 | 131.65ms | mfu 0.87%\n",
            "246 | loss 6.0459 | lr 1.230000e-04 | 127.93ms | mfu 0.87%\n",
            "247 | loss 5.9701 | lr 1.235000e-04 | 128.73ms | mfu 0.87%\n",
            "248 | loss 5.9540 | lr 1.240000e-04 | 129.53ms | mfu 0.87%\n",
            "249 | loss 5.9481 | lr 1.245000e-04 | 130.02ms | mfu 0.87%\n",
            "250 | loss 5.9630 | lr 1.250000e-04 | 140.73ms | mfu 0.86%\n",
            "251 | loss 5.9424 | lr 1.255000e-04 | 136.45ms | mfu 0.85%\n",
            "252 | loss 5.8674 | lr 1.260000e-04 | 137.26ms | mfu 0.85%\n",
            "253 | loss 5.9108 | lr 1.265000e-04 | 140.76ms | mfu 0.85%\n",
            "254 | loss 5.8924 | lr 1.270000e-04 | 135.28ms | mfu 0.84%\n",
            "255 | loss 5.8469 | lr 1.275000e-04 | 135.11ms | mfu 0.84%\n",
            "256 | loss 5.8614 | lr 1.280000e-04 | 136.04ms | mfu 0.84%\n",
            "257 | loss 5.8694 | lr 1.285000e-04 | 134.44ms | mfu 0.84%\n",
            "258 | loss 5.8102 | lr 1.290000e-04 | 140.16ms | mfu 0.83%\n",
            "259 | loss 5.7600 | lr 1.295000e-04 | 157.25ms | mfu 0.82%\n",
            "260 | loss 5.8738 | lr 1.300000e-04 | 165.64ms | mfu 0.81%\n",
            "261 | loss 5.8199 | lr 1.305000e-04 | 137.83ms | mfu 0.81%\n",
            "262 | loss 5.7522 | lr 1.310000e-04 | 137.07ms | mfu 0.81%\n",
            "263 | loss 5.7931 | lr 1.315000e-04 | 134.82ms | mfu 0.81%\n",
            "264 | loss 5.8038 | lr 1.320000e-04 | 138.54ms | mfu 0.81%\n",
            "265 | loss 5.7509 | lr 1.325000e-04 | 136.72ms | mfu 0.81%\n",
            "266 | loss 5.7051 | lr 1.330000e-04 | 135.56ms | mfu 0.81%\n",
            "267 | loss 5.7121 | lr 1.335000e-04 | 133.39ms | mfu 0.82%\n",
            "268 | loss 5.6952 | lr 1.340000e-04 | 134.59ms | mfu 0.82%\n",
            "269 | loss 5.7426 | lr 1.345000e-04 | 132.69ms | mfu 0.82%\n",
            "270 | loss 5.7135 | lr 1.350000e-04 | 137.14ms | mfu 0.82%\n",
            "271 | loss 5.7052 | lr 1.355000e-04 | 139.19ms | mfu 0.82%\n",
            "272 | loss 5.6878 | lr 1.360000e-04 | 136.49ms | mfu 0.82%\n",
            "273 | loss 5.6018 | lr 1.365000e-04 | 135.90ms | mfu 0.82%\n",
            "274 | loss 5.6644 | lr 1.370000e-04 | 138.38ms | mfu 0.82%\n",
            "275 | loss 5.6754 | lr 1.375000e-04 | 132.86ms | mfu 0.82%\n",
            "276 | loss 5.6660 | lr 1.380000e-04 | 131.39ms | mfu 0.82%\n",
            "277 | loss 5.6328 | lr 1.385000e-04 | 128.06ms | mfu 0.83%\n",
            "278 | loss 5.6204 | lr 1.390000e-04 | 128.79ms | mfu 0.83%\n",
            "279 | loss 5.6424 | lr 1.395000e-04 | 128.40ms | mfu 0.84%\n",
            "280 | loss 5.6112 | lr 1.400000e-04 | 130.80ms | mfu 0.84%\n",
            "281 | loss 5.5645 | lr 1.405000e-04 | 128.42ms | mfu 0.84%\n",
            "282 | loss 5.5911 | lr 1.410000e-04 | 129.17ms | mfu 0.84%\n",
            "283 | loss 5.5718 | lr 1.415000e-04 | 131.41ms | mfu 0.84%\n",
            "284 | loss 5.5411 | lr 1.420000e-04 | 130.20ms | mfu 0.85%\n",
            "285 | loss 5.5761 | lr 1.425000e-04 | 128.60ms | mfu 0.85%\n",
            "286 | loss 5.5454 | lr 1.430000e-04 | 128.49ms | mfu 0.85%\n",
            "287 | loss 5.5179 | lr 1.435000e-04 | 129.78ms | mfu 0.85%\n",
            "288 | loss 5.5202 | lr 1.440000e-04 | 129.00ms | mfu 0.85%\n",
            "289 | loss 5.4708 | lr 1.445000e-04 | 129.93ms | mfu 0.85%\n",
            "290 | loss 5.4548 | lr 1.450000e-04 | 130.47ms | mfu 0.85%\n",
            "291 | loss 5.4622 | lr 1.455000e-04 | 130.18ms | mfu 0.85%\n",
            "292 | loss 5.5109 | lr 1.460000e-04 | 131.77ms | mfu 0.85%\n",
            "293 | loss 5.4656 | lr 1.465000e-04 | 128.69ms | mfu 0.86%\n",
            "294 | loss 5.3692 | lr 1.470000e-04 | 130.60ms | mfu 0.86%\n",
            "295 | loss 5.4112 | lr 1.475000e-04 | 130.03ms | mfu 0.86%\n",
            "296 | loss 5.4851 | lr 1.480000e-04 | 129.01ms | mfu 0.86%\n",
            "297 | loss 5.4002 | lr 1.485000e-04 | 130.13ms | mfu 0.86%\n",
            "298 | loss 5.4073 | lr 1.490000e-04 | 130.22ms | mfu 0.86%\n",
            "299 | loss 5.4362 | lr 1.495000e-04 | 133.46ms | mfu 0.86%\n",
            "300 | loss 5.3676 | lr 1.500000e-04 | 132.58ms | mfu 0.85%\n",
            "301 | loss 5.4467 | lr 1.505000e-04 | 129.12ms | mfu 0.86%\n",
            "302 | loss 5.3597 | lr 1.510000e-04 | 129.70ms | mfu 0.86%\n",
            "303 | loss 5.3924 | lr 1.515000e-04 | 129.58ms | mfu 0.86%\n",
            "304 | loss 5.4870 | lr 1.520000e-04 | 129.71ms | mfu 0.86%\n",
            "305 | loss 5.3925 | lr 1.525000e-04 | 130.24ms | mfu 0.86%\n",
            "306 | loss 5.2508 | lr 1.530000e-04 | 130.16ms | mfu 0.86%\n",
            "307 | loss 5.2308 | lr 1.535000e-04 | 130.75ms | mfu 0.86%\n",
            "308 | loss 5.3268 | lr 1.540000e-04 | 128.62ms | mfu 0.86%\n",
            "309 | loss 5.3077 | lr 1.545000e-04 | 137.01ms | mfu 0.85%\n",
            "310 | loss 5.2724 | lr 1.550000e-04 | 128.64ms | mfu 0.86%\n",
            "311 | loss 5.1820 | lr 1.555000e-04 | 134.97ms | mfu 0.85%\n",
            "312 | loss 5.2344 | lr 1.560000e-04 | 130.52ms | mfu 0.85%\n",
            "313 | loss 5.1962 | lr 1.565000e-04 | 130.49ms | mfu 0.85%\n",
            "314 | loss 5.2515 | lr 1.570000e-04 | 129.82ms | mfu 0.85%\n",
            "315 | loss 5.2344 | lr 1.575000e-04 | 129.40ms | mfu 0.86%\n",
            "316 | loss 5.3154 | lr 1.580000e-04 | 128.63ms | mfu 0.86%\n",
            "317 | loss 5.2437 | lr 1.585000e-04 | 131.53ms | mfu 0.86%\n",
            "318 | loss 5.1937 | lr 1.590000e-04 | 128.83ms | mfu 0.86%\n",
            "319 | loss 5.1921 | lr 1.595000e-04 | 131.41ms | mfu 0.86%\n",
            "320 | loss 5.2374 | lr 1.600000e-04 | 130.72ms | mfu 0.86%\n",
            "321 | loss 5.2024 | lr 1.605000e-04 | 130.59ms | mfu 0.86%\n",
            "322 | loss 5.1895 | lr 1.610000e-04 | 130.12ms | mfu 0.86%\n",
            "323 | loss 5.1121 | lr 1.615000e-04 | 131.72ms | mfu 0.86%\n",
            "324 | loss 5.1248 | lr 1.620000e-04 | 131.90ms | mfu 0.86%\n",
            "325 | loss 5.1911 | lr 1.625000e-04 | 128.83ms | mfu 0.86%\n",
            "326 | loss 5.0969 | lr 1.630000e-04 | 130.72ms | mfu 0.86%\n",
            "327 | loss 5.1060 | lr 1.635000e-04 | 130.08ms | mfu 0.86%\n",
            "328 | loss 5.1334 | lr 1.640000e-04 | 129.57ms | mfu 0.86%\n",
            "329 | loss 5.0304 | lr 1.645000e-04 | 132.58ms | mfu 0.86%\n",
            "330 | loss 5.0741 | lr 1.650000e-04 | 130.46ms | mfu 0.86%\n",
            "331 | loss 5.0692 | lr 1.655000e-04 | 132.91ms | mfu 0.85%\n",
            "332 | loss 5.1524 | lr 1.660000e-04 | 131.46ms | mfu 0.85%\n",
            "333 | loss 5.0269 | lr 1.665000e-04 | 128.82ms | mfu 0.86%\n",
            "334 | loss 5.0762 | lr 1.670000e-04 | 129.36ms | mfu 0.86%\n",
            "335 | loss 5.0617 | lr 1.675000e-04 | 131.75ms | mfu 0.86%\n",
            "336 | loss 4.9873 | lr 1.680000e-04 | 129.38ms | mfu 0.86%\n",
            "337 | loss 4.9860 | lr 1.685000e-04 | 129.80ms | mfu 0.86%\n",
            "338 | loss 5.0359 | lr 1.690000e-04 | 134.26ms | mfu 0.85%\n",
            "339 | loss 4.9709 | lr 1.695000e-04 | 129.20ms | mfu 0.86%\n",
            "340 | loss 4.9208 | lr 1.700000e-04 | 129.59ms | mfu 0.86%\n",
            "341 | loss 4.9353 | lr 1.705000e-04 | 129.19ms | mfu 0.86%\n",
            "342 | loss 4.9638 | lr 1.710000e-04 | 135.75ms | mfu 0.85%\n",
            "343 | loss 5.0715 | lr 1.715000e-04 | 134.08ms | mfu 0.85%\n",
            "344 | loss 4.9543 | lr 1.720000e-04 | 131.82ms | mfu 0.85%\n",
            "345 | loss 4.9308 | lr 1.725000e-04 | 129.32ms | mfu 0.85%\n",
            "346 | loss 4.8815 | lr 1.730000e-04 | 130.21ms | mfu 0.85%\n",
            "347 | loss 4.9245 | lr 1.735000e-04 | 132.26ms | mfu 0.85%\n",
            "348 | loss 4.9126 | lr 1.740000e-04 | 130.04ms | mfu 0.85%\n",
            "349 | loss 4.9382 | lr 1.745000e-04 | 129.62ms | mfu 0.85%\n",
            "350 | loss 4.8795 | lr 1.750000e-04 | 130.45ms | mfu 0.85%\n",
            "351 | loss 4.9052 | lr 1.755000e-04 | 132.28ms | mfu 0.85%\n",
            "352 | loss 4.9041 | lr 1.760000e-04 | 140.82ms | mfu 0.85%\n",
            "353 | loss 4.8440 | lr 1.765000e-04 | 136.99ms | mfu 0.84%\n",
            "354 | loss 4.8860 | lr 1.770000e-04 | 143.39ms | mfu 0.84%\n",
            "355 | loss 4.7997 | lr 1.775000e-04 | 138.34ms | mfu 0.84%\n",
            "356 | loss 4.8654 | lr 1.780000e-04 | 135.78ms | mfu 0.83%\n",
            "357 | loss 4.8039 | lr 1.785000e-04 | 136.77ms | mfu 0.83%\n",
            "358 | loss 4.8400 | lr 1.790000e-04 | 137.59ms | mfu 0.83%\n",
            "359 | loss 4.8310 | lr 1.795000e-04 | 140.64ms | mfu 0.83%\n",
            "360 | loss 4.7820 | lr 1.800000e-04 | 137.96ms | mfu 0.83%\n",
            "361 | loss 4.8067 | lr 1.805000e-04 | 139.27ms | mfu 0.82%\n",
            "362 | loss 4.8176 | lr 1.810000e-04 | 142.52ms | mfu 0.82%\n",
            "363 | loss 4.8244 | lr 1.815000e-04 | 138.92ms | mfu 0.82%\n",
            "364 | loss 4.9121 | lr 1.820000e-04 | 137.80ms | mfu 0.82%\n",
            "365 | loss 4.6913 | lr 1.825000e-04 | 138.43ms | mfu 0.82%\n",
            "366 | loss 4.6980 | lr 1.830000e-04 | 137.00ms | mfu 0.82%\n",
            "367 | loss 4.7056 | lr 1.835000e-04 | 139.20ms | mfu 0.82%\n",
            "368 | loss 4.7522 | lr 1.840000e-04 | 139.67ms | mfu 0.81%\n",
            "369 | loss 4.7203 | lr 1.845000e-04 | 150.55ms | mfu 0.81%\n",
            "370 | loss 4.7102 | lr 1.850000e-04 | 133.41ms | mfu 0.81%\n",
            "371 | loss 4.6763 | lr 1.855000e-04 | 134.02ms | mfu 0.81%\n",
            "372 | loss 4.5976 | lr 1.860000e-04 | 136.67ms | mfu 0.81%\n",
            "373 | loss 4.6629 | lr 1.865000e-04 | 142.36ms | mfu 0.81%\n",
            "374 | loss 4.6786 | lr 1.870000e-04 | 138.12ms | mfu 0.81%\n",
            "375 | loss 4.6462 | lr 1.875000e-04 | 137.25ms | mfu 0.81%\n",
            "376 | loss 4.7173 | lr 1.880000e-04 | 140.10ms | mfu 0.81%\n",
            "377 | loss 4.6360 | lr 1.885000e-04 | 143.18ms | mfu 0.81%\n",
            "378 | loss 4.6305 | lr 1.890000e-04 | 137.94ms | mfu 0.81%\n",
            "379 | loss 4.6247 | lr 1.895000e-04 | 129.49ms | mfu 0.81%\n",
            "380 | loss 4.6006 | lr 1.900000e-04 | 132.51ms | mfu 0.82%\n",
            "381 | loss 4.6560 | lr 1.905000e-04 | 130.23ms | mfu 0.82%\n",
            "382 | loss 4.6143 | lr 1.910000e-04 | 130.16ms | mfu 0.82%\n",
            "383 | loss 4.6545 | lr 1.915000e-04 | 130.94ms | mfu 0.83%\n",
            "384 | loss 4.6330 | lr 1.920000e-04 | 133.37ms | mfu 0.83%\n",
            "385 | loss 4.5328 | lr 1.925000e-04 | 130.39ms | mfu 0.83%\n",
            "386 | loss 4.5412 | lr 1.930000e-04 | 129.71ms | mfu 0.83%\n",
            "387 | loss 4.5791 | lr 1.935000e-04 | 132.44ms | mfu 0.84%\n",
            "388 | loss 4.6023 | lr 1.940000e-04 | 129.68ms | mfu 0.84%\n",
            "389 | loss 4.6308 | lr 1.945000e-04 | 129.96ms | mfu 0.84%\n",
            "390 | loss 4.5046 | lr 1.950000e-04 | 132.82ms | mfu 0.84%\n",
            "391 | loss 4.5540 | lr 1.955000e-04 | 130.80ms | mfu 0.84%\n",
            "392 | loss 4.5829 | lr 1.960000e-04 | 133.05ms | mfu 0.84%\n",
            "393 | loss 4.6019 | lr 1.965000e-04 | 131.39ms | mfu 0.84%\n",
            "394 | loss 4.4832 | lr 1.970000e-04 | 130.04ms | mfu 0.84%\n",
            "395 | loss 4.5102 | lr 1.975000e-04 | 130.51ms | mfu 0.85%\n",
            "396 | loss 4.4829 | lr 1.980000e-04 | 133.42ms | mfu 0.85%\n",
            "397 | loss 4.4986 | lr 1.985000e-04 | 130.73ms | mfu 0.85%\n",
            "398 | loss 4.5314 | lr 1.990000e-04 | 131.39ms | mfu 0.85%\n",
            "399 | loss 4.5528 | lr 1.995000e-04 | 133.43ms | mfu 0.85%\n",
            "400 | loss 4.4864 | lr 2.000000e-04 | 132.47ms | mfu 0.85%\n",
            "401 | loss 4.4504 | lr 2.005000e-04 | 132.81ms | mfu 0.85%\n",
            "402 | loss 4.5259 | lr 2.010000e-04 | 130.29ms | mfu 0.85%\n",
            "403 | loss 4.4218 | lr 2.015000e-04 | 130.57ms | mfu 0.85%\n",
            "404 | loss 4.4373 | lr 2.020000e-04 | 130.84ms | mfu 0.85%\n",
            "405 | loss 4.4535 | lr 2.025000e-04 | 131.24ms | mfu 0.85%\n",
            "406 | loss 4.4089 | lr 2.030000e-04 | 130.20ms | mfu 0.85%\n",
            "407 | loss 4.4009 | lr 2.035000e-04 | 131.53ms | mfu 0.85%\n",
            "408 | loss 4.3719 | lr 2.040000e-04 | 132.81ms | mfu 0.85%\n",
            "409 | loss 4.4370 | lr 2.045000e-04 | 130.38ms | mfu 0.85%\n",
            "410 | loss 4.3778 | lr 2.050000e-04 | 131.10ms | mfu 0.85%\n",
            "411 | loss 4.4214 | lr 2.055000e-04 | 132.36ms | mfu 0.85%\n",
            "412 | loss 4.4039 | lr 2.060000e-04 | 130.65ms | mfu 0.85%\n",
            "413 | loss 4.3861 | lr 2.065000e-04 | 135.36ms | mfu 0.85%\n",
            "414 | loss 4.3086 | lr 2.070000e-04 | 131.13ms | mfu 0.85%\n",
            "415 | loss 4.3632 | lr 2.075000e-04 | 133.24ms | mfu 0.85%\n",
            "416 | loss 4.2983 | lr 2.080000e-04 | 130.91ms | mfu 0.85%\n",
            "417 | loss 4.2643 | lr 2.085000e-04 | 130.19ms | mfu 0.85%\n",
            "418 | loss 4.2719 | lr 2.090000e-04 | 130.99ms | mfu 0.85%\n",
            "419 | loss 4.2103 | lr 2.095000e-04 | 130.74ms | mfu 0.85%\n",
            "420 | loss 4.2787 | lr 2.100000e-04 | 130.80ms | mfu 0.85%\n",
            "421 | loss 4.2512 | lr 2.105000e-04 | 133.08ms | mfu 0.85%\n",
            "422 | loss 4.3016 | lr 2.110000e-04 | 130.82ms | mfu 0.85%\n",
            "423 | loss 4.2595 | lr 2.115000e-04 | 136.15ms | mfu 0.85%\n",
            "424 | loss 4.2468 | lr 2.120000e-04 | 132.46ms | mfu 0.85%\n",
            "425 | loss 4.3644 | lr 2.125000e-04 | 130.40ms | mfu 0.85%\n",
            "426 | loss 4.2520 | lr 2.130000e-04 | 134.50ms | mfu 0.85%\n",
            "427 | loss 4.2882 | lr 2.135000e-04 | 130.73ms | mfu 0.85%\n",
            "428 | loss 4.2212 | lr 2.140000e-04 | 130.52ms | mfu 0.85%\n",
            "429 | loss 4.2484 | lr 2.145000e-04 | 130.65ms | mfu 0.85%\n",
            "430 | loss 4.2027 | lr 2.150000e-04 | 130.80ms | mfu 0.85%\n",
            "431 | loss 4.3260 | lr 2.155000e-04 | 132.44ms | mfu 0.85%\n",
            "432 | loss 4.1821 | lr 2.160000e-04 | 131.56ms | mfu 0.85%\n",
            "433 | loss 4.1834 | lr 2.165000e-04 | 131.73ms | mfu 0.85%\n",
            "434 | loss 4.1696 | lr 2.170000e-04 | 131.43ms | mfu 0.85%\n",
            "435 | loss 4.1869 | lr 2.175000e-04 | 130.57ms | mfu 0.85%\n",
            "436 | loss 4.2408 | lr 2.180000e-04 | 135.10ms | mfu 0.85%\n",
            "437 | loss 4.2594 | lr 2.185000e-04 | 131.18ms | mfu 0.85%\n",
            "438 | loss 4.2524 | lr 2.190000e-04 | 132.00ms | mfu 0.85%\n",
            "439 | loss 4.2678 | lr 2.195000e-04 | 133.41ms | mfu 0.85%\n",
            "440 | loss 4.1954 | lr 2.200000e-04 | 130.64ms | mfu 0.85%\n",
            "441 | loss 4.1497 | lr 2.205000e-04 | 131.67ms | mfu 0.85%\n",
            "442 | loss 4.1711 | lr 2.210000e-04 | 130.63ms | mfu 0.85%\n",
            "443 | loss 4.1626 | lr 2.215000e-04 | 131.92ms | mfu 0.85%\n",
            "444 | loss 4.1529 | lr 2.220000e-04 | 132.16ms | mfu 0.85%\n",
            "445 | loss 4.0645 | lr 2.225000e-04 | 130.64ms | mfu 0.85%\n",
            "446 | loss 4.1122 | lr 2.230000e-04 | 133.70ms | mfu 0.85%\n",
            "447 | loss 4.2025 | lr 2.235000e-04 | 130.89ms | mfu 0.85%\n",
            "448 | loss 4.2130 | lr 2.240000e-04 | 131.00ms | mfu 0.85%\n",
            "449 | loss 4.0882 | lr 2.245000e-04 | 139.04ms | mfu 0.84%\n",
            "450 | loss 4.0618 | lr 2.250000e-04 | 130.19ms | mfu 0.85%\n",
            "451 | loss 4.2445 | lr 2.255000e-04 | 131.04ms | mfu 0.85%\n",
            "452 | loss 4.0977 | lr 2.260000e-04 | 132.87ms | mfu 0.85%\n",
            "453 | loss 4.0770 | lr 2.265000e-04 | 133.40ms | mfu 0.85%\n",
            "454 | loss 4.1960 | lr 2.270000e-04 | 145.89ms | mfu 0.84%\n",
            "455 | loss 4.1090 | lr 2.275000e-04 | 142.41ms | mfu 0.83%\n",
            "456 | loss 4.1058 | lr 2.280000e-04 | 142.38ms | mfu 0.83%\n",
            "457 | loss 3.9758 | lr 2.285000e-04 | 138.53ms | mfu 0.83%\n",
            "458 | loss 4.0080 | lr 2.290000e-04 | 140.58ms | mfu 0.82%\n",
            "459 | loss 4.0014 | lr 2.295000e-04 | 138.11ms | mfu 0.82%\n",
            "460 | loss 4.0586 | lr 2.300000e-04 | 138.58ms | mfu 0.82%\n",
            "461 | loss 4.0158 | lr 2.305000e-04 | 140.19ms | mfu 0.82%\n",
            "462 | loss 4.0190 | lr 2.310000e-04 | 137.50ms | mfu 0.82%\n",
            "463 | loss 4.1198 | lr 2.315000e-04 | 146.08ms | mfu 0.81%\n",
            "464 | loss 4.0233 | lr 2.320000e-04 | 139.44ms | mfu 0.81%\n",
            "465 | loss 3.9872 | lr 2.325000e-04 | 137.50ms | mfu 0.81%\n",
            "466 | loss 3.9580 | lr 2.330000e-04 | 140.00ms | mfu 0.81%\n",
            "467 | loss 4.0155 | lr 2.335000e-04 | 138.55ms | mfu 0.81%\n",
            "468 | loss 4.0560 | lr 2.340000e-04 | 140.32ms | mfu 0.81%\n",
            "469 | loss 4.0488 | lr 2.345000e-04 | 140.70ms | mfu 0.81%\n",
            "470 | loss 3.9218 | lr 2.350000e-04 | 140.61ms | mfu 0.81%\n",
            "471 | loss 3.9901 | lr 2.355000e-04 | 140.20ms | mfu 0.81%\n",
            "472 | loss 4.0962 | lr 2.360000e-04 | 140.28ms | mfu 0.80%\n",
            "473 | loss 3.9284 | lr 2.365000e-04 | 135.22ms | mfu 0.81%\n",
            "474 | loss 4.0227 | lr 2.370000e-04 | 138.31ms | mfu 0.81%\n",
            "475 | loss 4.0352 | lr 2.375000e-04 | 139.16ms | mfu 0.81%\n",
            "476 | loss 4.0047 | lr 2.380000e-04 | 146.66ms | mfu 0.80%\n",
            "477 | loss 4.0769 | lr 2.385000e-04 | 137.90ms | mfu 0.80%\n",
            "478 | loss 3.8800 | lr 2.390000e-04 | 136.65ms | mfu 0.80%\n",
            "479 | loss 4.0362 | lr 2.395000e-04 | 140.61ms | mfu 0.80%\n",
            "480 | loss 3.8816 | lr 2.400000e-04 | 130.21ms | mfu 0.81%\n",
            "481 | loss 4.0026 | lr 2.405000e-04 | 131.73ms | mfu 0.81%\n",
            "482 | loss 4.0031 | lr 2.410000e-04 | 133.90ms | mfu 0.82%\n",
            "483 | loss 3.9932 | lr 2.415000e-04 | 131.77ms | mfu 0.82%\n",
            "484 | loss 3.9665 | lr 2.420000e-04 | 132.46ms | mfu 0.82%\n",
            "485 | loss 3.9225 | lr 2.425000e-04 | 135.32ms | mfu 0.82%\n",
            "486 | loss 3.8704 | lr 2.430000e-04 | 131.80ms | mfu 0.82%\n",
            "487 | loss 3.9551 | lr 2.435000e-04 | 130.55ms | mfu 0.83%\n",
            "488 | loss 4.0171 | lr 2.440000e-04 | 131.25ms | mfu 0.83%\n",
            "489 | loss 3.8804 | lr 2.445000e-04 | 131.68ms | mfu 0.83%\n",
            "490 | loss 3.8234 | lr 2.450000e-04 | 131.76ms | mfu 0.83%\n",
            "491 | loss 3.9371 | lr 2.455000e-04 | 131.47ms | mfu 0.84%\n",
            "492 | loss 3.8537 | lr 2.460000e-04 | 130.91ms | mfu 0.84%\n",
            "493 | loss 4.0146 | lr 2.465000e-04 | 132.16ms | mfu 0.84%\n",
            "494 | loss 4.0209 | lr 2.470000e-04 | 133.50ms | mfu 0.84%\n",
            "495 | loss 3.8763 | lr 2.475000e-04 | 132.31ms | mfu 0.84%\n",
            "496 | loss 3.9447 | lr 2.480000e-04 | 131.56ms | mfu 0.84%\n",
            "497 | loss 3.8197 | lr 2.485000e-04 | 132.31ms | mfu 0.84%\n",
            "498 | loss 3.9231 | lr 2.490000e-04 | 131.83ms | mfu 0.84%\n",
            "499 | loss 3.8008 | lr 2.495000e-04 | 131.94ms | mfu 0.84%\n",
            "Created a PretokDataset with rng seed 42\n",
            "Created a PretokDataset with rng seed 42\n",
            "step 500: train loss 3.8627, val loss 3.8612\n",
            "saving checkpoint to out\n",
            "wrote out/model.bin\n",
            "500 | loss 3.8990 | lr 2.500000e-04 | 2640.29ms | mfu 0.76%\n",
            "501 | loss 3.8279 | lr 2.505000e-04 | 130.42ms | mfu 0.77%\n",
            "502 | loss 3.8014 | lr 2.510000e-04 | 131.28ms | mfu 0.78%\n",
            "503 | loss 3.8450 | lr 2.515000e-04 | 130.90ms | mfu 0.79%\n",
            "504 | loss 3.7992 | lr 2.520000e-04 | 135.86ms | mfu 0.79%\n",
            "505 | loss 3.8755 | lr 2.525000e-04 | 130.75ms | mfu 0.80%\n",
            "506 | loss 3.8949 | lr 2.530000e-04 | 135.95ms | mfu 0.80%\n",
            "507 | loss 3.7945 | lr 2.535000e-04 | 132.77ms | mfu 0.80%\n",
            "508 | loss 3.7978 | lr 2.540000e-04 | 132.23ms | mfu 0.81%\n",
            "509 | loss 3.7410 | lr 2.545000e-04 | 132.01ms | mfu 0.81%\n",
            "510 | loss 4.0058 | lr 2.550000e-04 | 137.23ms | mfu 0.81%\n",
            "511 | loss 3.8415 | lr 2.555000e-04 | 140.20ms | mfu 0.81%\n",
            "512 | loss 3.7060 | lr 2.560000e-04 | 130.04ms | mfu 0.82%\n",
            "513 | loss 3.8803 | lr 2.565000e-04 | 132.79ms | mfu 0.82%\n",
            "514 | loss 3.7383 | lr 2.570000e-04 | 133.48ms | mfu 0.82%\n",
            "515 | loss 3.8448 | lr 2.575000e-04 | 132.74ms | mfu 0.82%\n",
            "516 | loss 3.8355 | lr 2.580000e-04 | 133.03ms | mfu 0.82%\n",
            "517 | loss 3.8522 | lr 2.585000e-04 | 131.02ms | mfu 0.83%\n",
            "518 | loss 3.8638 | lr 2.590000e-04 | 131.33ms | mfu 0.83%\n",
            "519 | loss 3.7770 | lr 2.595000e-04 | 132.93ms | mfu 0.83%\n",
            "520 | loss 3.7187 | lr 2.600000e-04 | 131.04ms | mfu 0.83%\n",
            "521 | loss 3.7418 | lr 2.605000e-04 | 131.93ms | mfu 0.83%\n",
            "522 | loss 3.7699 | lr 2.610000e-04 | 132.31ms | mfu 0.84%\n",
            "523 | loss 3.7262 | lr 2.615000e-04 | 130.86ms | mfu 0.84%\n",
            "524 | loss 3.6474 | lr 2.620000e-04 | 130.64ms | mfu 0.84%\n",
            "525 | loss 3.7091 | lr 2.625000e-04 | 131.00ms | mfu 0.84%\n",
            "526 | loss 3.7184 | lr 2.630000e-04 | 132.90ms | mfu 0.84%\n",
            "527 | loss 3.8742 | lr 2.635000e-04 | 134.13ms | mfu 0.84%\n",
            "528 | loss 3.7458 | lr 2.640000e-04 | 130.49ms | mfu 0.84%\n",
            "529 | loss 3.7326 | lr 2.645000e-04 | 131.42ms | mfu 0.84%\n",
            "530 | loss 3.8232 | lr 2.650000e-04 | 134.52ms | mfu 0.84%\n",
            "531 | loss 3.7688 | lr 2.655000e-04 | 134.81ms | mfu 0.84%\n",
            "532 | loss 3.7660 | lr 2.660000e-04 | 130.03ms | mfu 0.84%\n",
            "533 | loss 3.7165 | lr 2.665000e-04 | 132.46ms | mfu 0.84%\n",
            "534 | loss 3.6668 | lr 2.670000e-04 | 132.82ms | mfu 0.84%\n",
            "535 | loss 3.7694 | lr 2.675000e-04 | 130.37ms | mfu 0.84%\n",
            "536 | loss 3.7031 | lr 2.680000e-04 | 138.68ms | mfu 0.84%\n",
            "537 | loss 3.6304 | lr 2.685000e-04 | 141.85ms | mfu 0.84%\n",
            "538 | loss 3.6776 | lr 2.690000e-04 | 137.29ms | mfu 0.83%\n",
            "539 | loss 3.7279 | lr 2.695000e-04 | 140.15ms | mfu 0.83%\n",
            "540 | loss 3.6327 | lr 2.700000e-04 | 137.90ms | mfu 0.83%\n",
            "541 | loss 3.6904 | lr 2.705000e-04 | 140.35ms | mfu 0.82%\n",
            "542 | loss 3.6347 | lr 2.710000e-04 | 146.56ms | mfu 0.82%\n",
            "543 | loss 3.7724 | lr 2.715000e-04 | 138.01ms | mfu 0.82%\n",
            "544 | loss 3.7509 | lr 2.720000e-04 | 142.34ms | mfu 0.81%\n",
            "545 | loss 3.6558 | lr 2.725000e-04 | 137.53ms | mfu 0.81%\n",
            "546 | loss 3.7114 | lr 2.730000e-04 | 137.85ms | mfu 0.81%\n",
            "547 | loss 3.7441 | lr 2.735000e-04 | 133.86ms | mfu 0.82%\n",
            "548 | loss 3.7326 | lr 2.740000e-04 | 139.68ms | mfu 0.81%\n",
            "549 | loss 3.5930 | lr 2.745000e-04 | 139.78ms | mfu 0.81%\n",
            "550 | loss 3.6016 | lr 2.750000e-04 | 141.64ms | mfu 0.81%\n",
            "551 | loss 3.6773 | lr 2.755000e-04 | 138.38ms | mfu 0.81%\n",
            "552 | loss 3.6557 | lr 2.760000e-04 | 140.86ms | mfu 0.81%\n",
            "553 | loss 3.5453 | lr 2.765000e-04 | 137.80ms | mfu 0.81%\n",
            "554 | loss 3.6974 | lr 2.770000e-04 | 138.58ms | mfu 0.81%\n",
            "555 | loss 3.6110 | lr 2.775000e-04 | 135.13ms | mfu 0.81%\n",
            "556 | loss 3.6211 | lr 2.780000e-04 | 136.83ms | mfu 0.81%\n",
            "557 | loss 3.6900 | lr 2.785000e-04 | 139.70ms | mfu 0.81%\n",
            "558 | loss 3.6029 | lr 2.790000e-04 | 142.43ms | mfu 0.81%\n",
            "559 | loss 3.5865 | lr 2.795000e-04 | 138.75ms | mfu 0.81%\n",
            "560 | loss 3.5592 | lr 2.800000e-04 | 139.77ms | mfu 0.81%\n",
            "561 | loss 3.5490 | lr 2.805000e-04 | 141.63ms | mfu 0.81%\n",
            "562 | loss 3.6577 | lr 2.810000e-04 | 131.54ms | mfu 0.81%\n",
            "563 | loss 3.6708 | lr 2.815000e-04 | 131.72ms | mfu 0.81%\n",
            "564 | loss 3.5824 | lr 2.820000e-04 | 132.82ms | mfu 0.82%\n",
            "565 | loss 3.5708 | lr 2.825000e-04 | 132.67ms | mfu 0.82%\n",
            "566 | loss 3.6196 | lr 2.830000e-04 | 129.73ms | mfu 0.82%\n",
            "567 | loss 3.5690 | lr 2.835000e-04 | 130.90ms | mfu 0.83%\n",
            "568 | loss 3.6202 | lr 2.840000e-04 | 131.12ms | mfu 0.83%\n",
            "569 | loss 3.6268 | lr 2.845000e-04 | 130.91ms | mfu 0.83%\n",
            "570 | loss 3.6083 | lr 2.850000e-04 | 130.65ms | mfu 0.83%\n",
            "571 | loss 3.6344 | lr 2.855000e-04 | 132.89ms | mfu 0.83%\n",
            "572 | loss 3.5672 | lr 2.860000e-04 | 133.57ms | mfu 0.84%\n",
            "573 | loss 3.6290 | lr 2.865000e-04 | 129.73ms | mfu 0.84%\n",
            "574 | loss 3.6264 | lr 2.870000e-04 | 131.23ms | mfu 0.84%\n",
            "575 | loss 3.5266 | lr 2.875000e-04 | 131.37ms | mfu 0.84%\n",
            "576 | loss 3.5418 | lr 2.880000e-04 | 130.81ms | mfu 0.84%\n",
            "577 | loss 3.5110 | lr 2.885000e-04 | 130.80ms | mfu 0.84%\n",
            "578 | loss 3.5990 | lr 2.890000e-04 | 133.08ms | mfu 0.84%\n",
            "579 | loss 3.5216 | lr 2.895000e-04 | 130.28ms | mfu 0.84%\n",
            "580 | loss 3.5019 | lr 2.900000e-04 | 133.83ms | mfu 0.84%\n",
            "581 | loss 3.4656 | lr 2.905000e-04 | 133.12ms | mfu 0.84%\n",
            "582 | loss 3.5649 | lr 2.910000e-04 | 130.83ms | mfu 0.84%\n",
            "583 | loss 3.5447 | lr 2.915000e-04 | 131.03ms | mfu 0.85%\n",
            "584 | loss 3.5267 | lr 2.920000e-04 | 136.30ms | mfu 0.84%\n",
            "585 | loss 3.6456 | lr 2.925000e-04 | 132.05ms | mfu 0.84%\n",
            "586 | loss 3.7112 | lr 2.930000e-04 | 131.34ms | mfu 0.84%\n",
            "587 | loss 3.4691 | lr 2.935000e-04 | 130.51ms | mfu 0.85%\n",
            "588 | loss 3.6298 | lr 2.940000e-04 | 132.97ms | mfu 0.85%\n",
            "589 | loss 3.4227 | lr 2.945000e-04 | 129.77ms | mfu 0.85%\n",
            "590 | loss 3.5124 | lr 2.950000e-04 | 130.27ms | mfu 0.85%\n",
            "591 | loss 3.5890 | lr 2.955000e-04 | 132.53ms | mfu 0.85%\n",
            "592 | loss 3.4644 | lr 2.960000e-04 | 129.93ms | mfu 0.85%\n",
            "593 | loss 3.5262 | lr 2.965000e-04 | 130.50ms | mfu 0.85%\n",
            "594 | loss 3.5741 | lr 2.970000e-04 | 131.90ms | mfu 0.85%\n",
            "595 | loss 3.5005 | lr 2.975000e-04 | 132.47ms | mfu 0.85%\n",
            "596 | loss 3.5683 | lr 2.980000e-04 | 133.21ms | mfu 0.85%\n",
            "597 | loss 3.5374 | lr 2.985000e-04 | 130.43ms | mfu 0.85%\n",
            "598 | loss 3.4301 | lr 2.990000e-04 | 129.83ms | mfu 0.85%\n",
            "599 | loss 3.5748 | lr 2.995000e-04 | 130.59ms | mfu 0.85%\n",
            "600 | loss 3.5189 | lr 3.000000e-04 | 130.03ms | mfu 0.85%\n",
            "601 | loss 3.5355 | lr 3.005000e-04 | 131.97ms | mfu 0.85%\n",
            "602 | loss 3.4671 | lr 3.010000e-04 | 129.92ms | mfu 0.85%\n",
            "603 | loss 3.4506 | lr 3.015000e-04 | 133.25ms | mfu 0.85%\n",
            "604 | loss 3.4622 | lr 3.020000e-04 | 133.08ms | mfu 0.85%\n",
            "605 | loss 3.5067 | lr 3.025000e-04 | 131.01ms | mfu 0.85%\n",
            "606 | loss 3.4535 | lr 3.030000e-04 | 129.93ms | mfu 0.85%\n",
            "607 | loss 3.3859 | lr 3.035000e-04 | 135.60ms | mfu 0.85%\n",
            "608 | loss 3.5190 | lr 3.040000e-04 | 129.67ms | mfu 0.85%\n",
            "609 | loss 3.4905 | lr 3.045000e-04 | 130.80ms | mfu 0.85%\n",
            "610 | loss 3.4607 | lr 3.050000e-04 | 130.04ms | mfu 0.85%\n",
            "611 | loss 3.4555 | lr 3.055000e-04 | 133.66ms | mfu 0.85%\n",
            "612 | loss 3.5529 | lr 3.060000e-04 | 130.68ms | mfu 0.85%\n",
            "613 | loss 3.4553 | lr 3.065000e-04 | 130.23ms | mfu 0.85%\n",
            "614 | loss 3.4244 | lr 3.070000e-04 | 131.77ms | mfu 0.85%\n",
            "615 | loss 3.4233 | lr 3.075000e-04 | 131.62ms | mfu 0.85%\n",
            "616 | loss 3.4272 | lr 3.080000e-04 | 131.55ms | mfu 0.85%\n",
            "617 | loss 3.4562 | lr 3.085000e-04 | 131.15ms | mfu 0.85%\n",
            "618 | loss 3.3440 | lr 3.090000e-04 | 131.18ms | mfu 0.85%\n",
            "619 | loss 3.3952 | lr 3.095000e-04 | 132.51ms | mfu 0.85%\n",
            "620 | loss 3.5047 | lr 3.100000e-04 | 130.18ms | mfu 0.85%\n",
            "621 | loss 3.5227 | lr 3.105000e-04 | 130.23ms | mfu 0.85%\n",
            "622 | loss 3.4487 | lr 3.110000e-04 | 130.84ms | mfu 0.85%\n",
            "623 | loss 3.4446 | lr 3.115000e-04 | 130.94ms | mfu 0.85%\n",
            "624 | loss 3.5520 | lr 3.120000e-04 | 131.57ms | mfu 0.85%\n",
            "625 | loss 3.4068 | lr 3.125000e-04 | 131.91ms | mfu 0.85%\n",
            "626 | loss 3.4554 | lr 3.130000e-04 | 130.15ms | mfu 0.85%\n",
            "627 | loss 3.3945 | lr 3.135000e-04 | 133.67ms | mfu 0.85%\n",
            "628 | loss 3.4587 | lr 3.140000e-04 | 131.99ms | mfu 0.85%\n",
            "629 | loss 3.3738 | lr 3.145000e-04 | 130.58ms | mfu 0.85%\n",
            "630 | loss 3.4992 | lr 3.150000e-04 | 129.68ms | mfu 0.85%\n",
            "631 | loss 3.4698 | lr 3.155000e-04 | 129.70ms | mfu 0.85%\n",
            "632 | loss 3.3105 | lr 3.160000e-04 | 130.20ms | mfu 0.85%\n",
            "633 | loss 3.4139 | lr 3.165000e-04 | 129.99ms | mfu 0.85%\n",
            "634 | loss 3.3079 | lr 3.170000e-04 | 131.55ms | mfu 0.85%\n",
            "635 | loss 3.3907 | lr 3.175000e-04 | 130.12ms | mfu 0.85%\n",
            "636 | loss 3.4261 | lr 3.180000e-04 | 129.06ms | mfu 0.86%\n",
            "637 | loss 3.3571 | lr 3.185000e-04 | 130.38ms | mfu 0.86%\n",
            "638 | loss 3.3434 | lr 3.190000e-04 | 140.60ms | mfu 0.85%\n",
            "639 | loss 3.4335 | lr 3.195000e-04 | 140.72ms | mfu 0.84%\n",
            "640 | loss 3.2599 | lr 3.200000e-04 | 141.49ms | mfu 0.84%\n",
            "641 | loss 3.3714 | lr 3.205000e-04 | 137.46ms | mfu 0.84%\n",
            "642 | loss 3.3381 | lr 3.210000e-04 | 141.22ms | mfu 0.83%\n",
            "643 | loss 3.3490 | lr 3.215000e-04 | 141.43ms | mfu 0.83%\n",
            "644 | loss 3.3266 | lr 3.220000e-04 | 137.95ms | mfu 0.83%\n",
            "645 | loss 3.3502 | lr 3.225000e-04 | 141.50ms | mfu 0.82%\n",
            "646 | loss 3.3595 | lr 3.230000e-04 | 138.79ms | mfu 0.82%\n",
            "647 | loss 3.4051 | lr 3.235000e-04 | 136.43ms | mfu 0.82%\n",
            "648 | loss 3.3112 | lr 3.240000e-04 | 134.30ms | mfu 0.82%\n",
            "649 | loss 3.3846 | lr 3.245000e-04 | 136.25ms | mfu 0.82%\n",
            "650 | loss 3.3428 | lr 3.250000e-04 | 137.31ms | mfu 0.82%\n",
            "651 | loss 3.2592 | lr 3.255000e-04 | 137.46ms | mfu 0.82%\n",
            "652 | loss 3.3319 | lr 3.260000e-04 | 136.08ms | mfu 0.82%\n",
            "653 | loss 3.3711 | lr 3.265000e-04 | 136.38ms | mfu 0.82%\n",
            "654 | loss 3.3073 | lr 3.270000e-04 | 141.98ms | mfu 0.82%\n",
            "655 | loss 3.3338 | lr 3.275000e-04 | 137.76ms | mfu 0.82%\n",
            "656 | loss 3.3393 | lr 3.280000e-04 | 135.88ms | mfu 0.82%\n",
            "657 | loss 3.2933 | lr 3.285000e-04 | 142.36ms | mfu 0.81%\n",
            "658 | loss 3.2823 | lr 3.290000e-04 | 134.01ms | mfu 0.82%\n",
            "659 | loss 3.4095 | lr 3.295000e-04 | 137.76ms | mfu 0.82%\n",
            "660 | loss 3.2958 | lr 3.300000e-04 | 135.58ms | mfu 0.82%\n",
            "661 | loss 3.2633 | lr 3.305000e-04 | 142.30ms | mfu 0.81%\n",
            "662 | loss 3.3588 | lr 3.310000e-04 | 153.97ms | mfu 0.80%\n",
            "663 | loss 3.3092 | lr 3.315000e-04 | 140.65ms | mfu 0.80%\n",
            "664 | loss 3.4042 | lr 3.320000e-04 | 130.06ms | mfu 0.81%\n",
            "665 | loss 3.3696 | lr 3.325000e-04 | 132.51ms | mfu 0.81%\n",
            "666 | loss 3.3335 | lr 3.330000e-04 | 129.24ms | mfu 0.82%\n",
            "667 | loss 3.3725 | lr 3.335000e-04 | 129.99ms | mfu 0.82%\n",
            "668 | loss 3.1796 | lr 3.340000e-04 | 130.53ms | mfu 0.83%\n",
            "669 | loss 3.3678 | lr 3.345000e-04 | 130.03ms | mfu 0.83%\n",
            "670 | loss 3.4005 | lr 3.350000e-04 | 131.74ms | mfu 0.83%\n",
            "671 | loss 3.3454 | lr 3.355000e-04 | 133.11ms | mfu 0.83%\n",
            "672 | loss 3.4209 | lr 3.360000e-04 | 128.35ms | mfu 0.84%\n",
            "673 | loss 3.2944 | lr 3.365000e-04 | 129.34ms | mfu 0.84%\n",
            "674 | loss 3.2556 | lr 3.370000e-04 | 129.43ms | mfu 0.84%\n",
            "675 | loss 3.2518 | lr 3.375000e-04 | 131.69ms | mfu 0.84%\n",
            "676 | loss 3.2342 | lr 3.380000e-04 | 131.65ms | mfu 0.84%\n",
            "677 | loss 3.2347 | lr 3.385000e-04 | 131.71ms | mfu 0.84%\n",
            "678 | loss 3.3060 | lr 3.390000e-04 | 130.42ms | mfu 0.85%\n",
            "679 | loss 3.4295 | lr 3.395000e-04 | 130.17ms | mfu 0.85%\n",
            "680 | loss 3.2560 | lr 3.400000e-04 | 133.02ms | mfu 0.85%\n",
            "681 | loss 3.2567 | lr 3.405000e-04 | 131.46ms | mfu 0.85%\n",
            "682 | loss 3.2666 | lr 3.410000e-04 | 128.72ms | mfu 0.85%\n",
            "683 | loss 3.1702 | lr 3.415000e-04 | 134.66ms | mfu 0.85%\n",
            "684 | loss 3.3588 | lr 3.420000e-04 | 129.84ms | mfu 0.85%\n",
            "685 | loss 3.1529 | lr 3.425000e-04 | 129.74ms | mfu 0.85%\n",
            "686 | loss 3.1214 | lr 3.430000e-04 | 129.67ms | mfu 0.85%\n",
            "687 | loss 3.3065 | lr 3.435000e-04 | 130.38ms | mfu 0.85%\n",
            "688 | loss 3.3193 | lr 3.440000e-04 | 134.03ms | mfu 0.85%\n",
            "689 | loss 3.2877 | lr 3.445000e-04 | 128.23ms | mfu 0.85%\n",
            "690 | loss 3.3706 | lr 3.450000e-04 | 132.46ms | mfu 0.85%\n",
            "691 | loss 3.2662 | lr 3.455000e-04 | 133.46ms | mfu 0.85%\n",
            "692 | loss 3.2845 | lr 3.460000e-04 | 130.20ms | mfu 0.85%\n",
            "693 | loss 3.2747 | lr 3.465000e-04 | 130.58ms | mfu 0.85%\n",
            "694 | loss 3.1656 | lr 3.470000e-04 | 129.67ms | mfu 0.85%\n",
            "695 | loss 3.2414 | lr 3.475000e-04 | 131.92ms | mfu 0.85%\n",
            "696 | loss 3.2295 | lr 3.480000e-04 | 133.04ms | mfu 0.85%\n",
            "697 | loss 3.1822 | lr 3.485000e-04 | 130.17ms | mfu 0.85%\n",
            "698 | loss 3.1801 | lr 3.490000e-04 | 131.55ms | mfu 0.85%\n",
            "699 | loss 3.2321 | lr 3.495000e-04 | 129.44ms | mfu 0.85%\n",
            "700 | loss 3.1645 | lr 3.500000e-04 | 131.98ms | mfu 0.85%\n",
            "701 | loss 3.2988 | lr 3.505000e-04 | 169.12ms | mfu 0.83%\n",
            "702 | loss 3.3300 | lr 3.510000e-04 | 136.18ms | mfu 0.83%\n",
            "703 | loss 3.2221 | lr 3.515000e-04 | 132.37ms | mfu 0.83%\n",
            "704 | loss 3.1997 | lr 3.520000e-04 | 129.36ms | mfu 0.84%\n",
            "705 | loss 3.1087 | lr 3.525000e-04 | 129.13ms | mfu 0.84%\n",
            "706 | loss 3.2006 | lr 3.530000e-04 | 131.68ms | mfu 0.84%\n",
            "707 | loss 3.2502 | lr 3.535000e-04 | 130.06ms | mfu 0.84%\n",
            "708 | loss 3.1284 | lr 3.540000e-04 | 131.07ms | mfu 0.84%\n",
            "709 | loss 3.2130 | lr 3.545000e-04 | 130.96ms | mfu 0.84%\n",
            "710 | loss 3.2856 | lr 3.550000e-04 | 129.56ms | mfu 0.85%\n",
            "711 | loss 3.2292 | lr 3.555000e-04 | 134.68ms | mfu 0.84%\n",
            "712 | loss 3.3290 | lr 3.560000e-04 | 131.88ms | mfu 0.85%\n",
            "713 | loss 3.3218 | lr 3.565000e-04 | 129.81ms | mfu 0.85%\n",
            "714 | loss 3.1623 | lr 3.570000e-04 | 129.22ms | mfu 0.85%\n",
            "715 | loss 3.3099 | lr 3.575000e-04 | 129.00ms | mfu 0.85%\n",
            "716 | loss 3.2561 | lr 3.580000e-04 | 131.84ms | mfu 0.85%\n",
            "717 | loss 3.2152 | lr 3.585000e-04 | 130.64ms | mfu 0.85%\n",
            "718 | loss 3.1973 | lr 3.590000e-04 | 129.24ms | mfu 0.85%\n",
            "719 | loss 3.2130 | lr 3.595000e-04 | 135.40ms | mfu 0.85%\n",
            "720 | loss 3.1090 | lr 3.600000e-04 | 129.82ms | mfu 0.85%\n",
            "721 | loss 3.1773 | lr 3.605000e-04 | 130.66ms | mfu 0.85%\n",
            "722 | loss 3.1524 | lr 3.610000e-04 | 130.97ms | mfu 0.85%\n",
            "723 | loss 3.2380 | lr 3.615000e-04 | 131.66ms | mfu 0.85%\n",
            "724 | loss 3.1625 | lr 3.620000e-04 | 129.75ms | mfu 0.85%\n",
            "725 | loss 3.2072 | lr 3.625000e-04 | 129.30ms | mfu 0.85%\n",
            "726 | loss 3.2533 | lr 3.630000e-04 | 131.33ms | mfu 0.85%\n",
            "727 | loss 3.2481 | lr 3.635000e-04 | 130.39ms | mfu 0.85%\n",
            "728 | loss 3.1172 | lr 3.640000e-04 | 129.84ms | mfu 0.85%\n",
            "729 | loss 3.1143 | lr 3.645000e-04 | 131.34ms | mfu 0.85%\n",
            "730 | loss 3.1944 | lr 3.650000e-04 | 128.99ms | mfu 0.86%\n",
            "731 | loss 3.1816 | lr 3.655000e-04 | 129.92ms | mfu 0.86%\n",
            "732 | loss 3.2416 | lr 3.660000e-04 | 130.96ms | mfu 0.86%\n",
            "733 | loss 3.1530 | lr 3.665000e-04 | 129.67ms | mfu 0.86%\n",
            "734 | loss 3.1733 | lr 3.670000e-04 | 130.82ms | mfu 0.86%\n",
            "735 | loss 3.2555 | lr 3.675000e-04 | 130.49ms | mfu 0.86%\n",
            "736 | loss 3.1083 | lr 3.680000e-04 | 132.11ms | mfu 0.86%\n",
            "737 | loss 3.2306 | lr 3.685000e-04 | 130.88ms | mfu 0.86%\n",
            "738 | loss 3.1859 | lr 3.690000e-04 | 128.99ms | mfu 0.86%\n",
            "739 | loss 3.1295 | lr 3.695000e-04 | 131.57ms | mfu 0.86%\n",
            "740 | loss 3.0994 | lr 3.700000e-04 | 145.43ms | mfu 0.85%\n",
            "741 | loss 3.1005 | lr 3.705000e-04 | 137.94ms | mfu 0.84%\n",
            "742 | loss 3.2457 | lr 3.710000e-04 | 139.03ms | mfu 0.84%\n",
            "743 | loss 3.1455 | lr 3.715000e-04 | 134.67ms | mfu 0.84%\n",
            "744 | loss 3.1769 | lr 3.720000e-04 | 136.35ms | mfu 0.84%\n",
            "745 | loss 2.9939 | lr 3.725000e-04 | 135.62ms | mfu 0.84%\n",
            "746 | loss 3.1531 | lr 3.730000e-04 | 139.66ms | mfu 0.83%\n",
            "747 | loss 3.1608 | lr 3.735000e-04 | 137.91ms | mfu 0.83%\n",
            "748 | loss 3.1720 | lr 3.740000e-04 | 136.27ms | mfu 0.83%\n",
            "749 | loss 3.0569 | lr 3.745000e-04 | 133.38ms | mfu 0.83%\n",
            "750 | loss 3.1583 | lr 3.750000e-04 | 136.83ms | mfu 0.83%\n",
            "751 | loss 3.2989 | lr 3.755000e-04 | 133.76ms | mfu 0.83%\n",
            "752 | loss 3.1630 | lr 3.760000e-04 | 136.97ms | mfu 0.83%\n",
            "753 | loss 3.1742 | lr 3.765000e-04 | 138.01ms | mfu 0.83%\n",
            "754 | loss 3.1323 | lr 3.770000e-04 | 138.00ms | mfu 0.82%\n",
            "755 | loss 3.0851 | lr 3.775000e-04 | 135.75ms | mfu 0.82%\n",
            "756 | loss 3.0103 | lr 3.780000e-04 | 136.16ms | mfu 0.82%\n",
            "757 | loss 3.1151 | lr 3.785000e-04 | 135.05ms | mfu 0.82%\n",
            "758 | loss 3.1123 | lr 3.790000e-04 | 139.75ms | mfu 0.82%\n",
            "759 | loss 3.1007 | lr 3.795000e-04 | 134.84ms | mfu 0.82%\n",
            "760 | loss 3.1396 | lr 3.800000e-04 | 134.25ms | mfu 0.82%\n",
            "761 | loss 3.2383 | lr 3.805000e-04 | 142.24ms | mfu 0.82%\n",
            "762 | loss 3.0164 | lr 3.810000e-04 | 141.31ms | mfu 0.82%\n",
            "763 | loss 3.1554 | lr 3.815000e-04 | 139.17ms | mfu 0.82%\n",
            "764 | loss 3.1221 | lr 3.820000e-04 | 142.72ms | mfu 0.81%\n",
            "765 | loss 3.1205 | lr 3.825000e-04 | 140.95ms | mfu 0.81%\n",
            "766 | loss 3.1504 | lr 3.830000e-04 | 138.82ms | mfu 0.81%\n",
            "767 | loss 3.0969 | lr 3.835000e-04 | 129.74ms | mfu 0.82%\n",
            "768 | loss 3.2104 | lr 3.840000e-04 | 129.76ms | mfu 0.82%\n",
            "769 | loss 3.0642 | lr 3.845000e-04 | 129.43ms | mfu 0.82%\n",
            "770 | loss 3.1138 | lr 3.850000e-04 | 128.96ms | mfu 0.83%\n",
            "771 | loss 3.0214 | lr 3.855000e-04 | 128.80ms | mfu 0.83%\n",
            "772 | loss 3.0616 | lr 3.860000e-04 | 129.17ms | mfu 0.84%\n",
            "773 | loss 3.1288 | lr 3.865000e-04 | 131.29ms | mfu 0.84%\n",
            "774 | loss 3.1481 | lr 3.870000e-04 | 132.39ms | mfu 0.84%\n",
            "775 | loss 3.1246 | lr 3.875000e-04 | 129.57ms | mfu 0.84%\n",
            "776 | loss 3.0321 | lr 3.880000e-04 | 129.89ms | mfu 0.84%\n",
            "777 | loss 3.0719 | lr 3.885000e-04 | 129.07ms | mfu 0.85%\n",
            "778 | loss 3.0716 | lr 3.890000e-04 | 130.93ms | mfu 0.85%\n",
            "779 | loss 3.2204 | lr 3.895000e-04 | 129.31ms | mfu 0.85%\n",
            "780 | loss 3.0961 | lr 3.900000e-04 | 128.85ms | mfu 0.85%\n",
            "781 | loss 3.1072 | lr 3.905000e-04 | 133.83ms | mfu 0.85%\n",
            "782 | loss 2.9694 | lr 3.910000e-04 | 128.77ms | mfu 0.85%\n",
            "783 | loss 3.0146 | lr 3.915000e-04 | 128.66ms | mfu 0.85%\n",
            "784 | loss 3.1017 | lr 3.920000e-04 | 131.19ms | mfu 0.85%\n",
            "785 | loss 3.0864 | lr 3.925000e-04 | 129.21ms | mfu 0.85%\n",
            "786 | loss 3.0822 | lr 3.930000e-04 | 130.43ms | mfu 0.85%\n",
            "787 | loss 3.0246 | lr 3.935000e-04 | 129.06ms | mfu 0.86%\n",
            "788 | loss 3.0103 | lr 3.940000e-04 | 131.92ms | mfu 0.85%\n",
            "789 | loss 3.0669 | lr 3.945000e-04 | 134.52ms | mfu 0.85%\n",
            "790 | loss 3.0122 | lr 3.950000e-04 | 129.45ms | mfu 0.85%\n",
            "791 | loss 3.0589 | lr 3.955000e-04 | 129.14ms | mfu 0.85%\n",
            "792 | loss 3.1118 | lr 3.960000e-04 | 132.16ms | mfu 0.85%\n",
            "793 | loss 2.9857 | lr 3.965000e-04 | 128.79ms | mfu 0.86%\n",
            "794 | loss 3.0657 | lr 3.970000e-04 | 128.53ms | mfu 0.86%\n",
            "795 | loss 3.0198 | lr 3.975000e-04 | 131.75ms | mfu 0.86%\n",
            "796 | loss 3.0653 | lr 3.980000e-04 | 130.48ms | mfu 0.86%\n",
            "797 | loss 3.2084 | lr 3.985000e-04 | 128.43ms | mfu 0.86%\n",
            "798 | loss 2.8978 | lr 3.990000e-04 | 129.53ms | mfu 0.86%\n",
            "799 | loss 3.0514 | lr 3.995000e-04 | 129.50ms | mfu 0.86%\n",
            "800 | loss 3.0842 | lr 4.000000e-04 | 129.25ms | mfu 0.86%\n",
            "801 | loss 2.9799 | lr 4.005000e-04 | 136.43ms | mfu 0.86%\n",
            "802 | loss 3.1190 | lr 4.010000e-04 | 129.80ms | mfu 0.86%\n",
            "803 | loss 2.9930 | lr 4.015000e-04 | 128.62ms | mfu 0.86%\n",
            "804 | loss 3.0788 | lr 4.020000e-04 | 132.65ms | mfu 0.86%\n",
            "805 | loss 3.0180 | lr 4.025000e-04 | 129.61ms | mfu 0.86%\n",
            "806 | loss 3.0160 | lr 4.030000e-04 | 128.17ms | mfu 0.86%\n",
            "807 | loss 2.9093 | lr 4.035000e-04 | 129.33ms | mfu 0.86%\n",
            "808 | loss 3.0424 | lr 4.040000e-04 | 130.31ms | mfu 0.86%\n",
            "809 | loss 3.0087 | lr 4.045000e-04 | 129.82ms | mfu 0.86%\n",
            "810 | loss 3.1520 | lr 4.050000e-04 | 128.24ms | mfu 0.86%\n",
            "811 | loss 2.8613 | lr 4.055000e-04 | 135.70ms | mfu 0.86%\n",
            "812 | loss 2.9520 | lr 4.060000e-04 | 135.13ms | mfu 0.85%\n",
            "813 | loss 3.0435 | lr 4.065000e-04 | 128.66ms | mfu 0.86%\n",
            "814 | loss 3.0245 | lr 4.070000e-04 | 128.79ms | mfu 0.86%\n",
            "815 | loss 2.9919 | lr 4.075000e-04 | 129.53ms | mfu 0.86%\n",
            "816 | loss 3.0051 | lr 4.080000e-04 | 131.03ms | mfu 0.86%\n",
            "817 | loss 3.0813 | lr 4.085000e-04 | 128.44ms | mfu 0.86%\n",
            "818 | loss 2.9438 | lr 4.090000e-04 | 129.31ms | mfu 0.86%\n",
            "819 | loss 3.0602 | lr 4.095000e-04 | 129.61ms | mfu 0.86%\n",
            "820 | loss 3.0869 | lr 4.100000e-04 | 129.07ms | mfu 0.86%\n",
            "821 | loss 3.0626 | lr 4.105000e-04 | 128.31ms | mfu 0.86%\n",
            "822 | loss 3.0511 | lr 4.110000e-04 | 130.50ms | mfu 0.86%\n",
            "823 | loss 2.8824 | lr 4.115000e-04 | 130.14ms | mfu 0.86%\n",
            "824 | loss 3.0309 | lr 4.120000e-04 | 128.58ms | mfu 0.86%\n",
            "825 | loss 2.9686 | lr 4.125000e-04 | 131.25ms | mfu 0.86%\n",
            "826 | loss 2.9219 | lr 4.130000e-04 | 128.98ms | mfu 0.86%\n",
            "827 | loss 2.9724 | lr 4.135000e-04 | 131.84ms | mfu 0.86%\n",
            "828 | loss 2.9888 | lr 4.140000e-04 | 131.55ms | mfu 0.86%\n",
            "829 | loss 2.9664 | lr 4.145000e-04 | 131.36ms | mfu 0.86%\n",
            "830 | loss 2.9533 | lr 4.150000e-04 | 129.05ms | mfu 0.86%\n",
            "831 | loss 3.0832 | lr 4.155000e-04 | 128.67ms | mfu 0.86%\n",
            "832 | loss 3.0374 | lr 4.160000e-04 | 129.18ms | mfu 0.86%\n",
            "833 | loss 2.9486 | lr 4.165000e-04 | 131.95ms | mfu 0.86%\n",
            "834 | loss 2.9762 | lr 4.170000e-04 | 129.71ms | mfu 0.86%\n",
            "835 | loss 3.0128 | lr 4.175000e-04 | 131.39ms | mfu 0.86%\n",
            "836 | loss 2.9249 | lr 4.180000e-04 | 133.05ms | mfu 0.86%\n",
            "837 | loss 3.0277 | lr 4.185000e-04 | 129.55ms | mfu 0.86%\n",
            "838 | loss 2.9215 | lr 4.190000e-04 | 129.73ms | mfu 0.86%\n",
            "839 | loss 2.9898 | lr 4.195000e-04 | 128.77ms | mfu 0.86%\n",
            "840 | loss 3.0417 | lr 4.200000e-04 | 135.11ms | mfu 0.86%\n",
            "841 | loss 3.0226 | lr 4.205000e-04 | 128.62ms | mfu 0.86%\n",
            "842 | loss 2.9756 | lr 4.210000e-04 | 131.44ms | mfu 0.86%\n",
            "843 | loss 2.8719 | lr 4.215000e-04 | 146.33ms | mfu 0.85%\n",
            "844 | loss 2.9879 | lr 4.220000e-04 | 136.94ms | mfu 0.84%\n",
            "845 | loss 3.0326 | lr 4.225000e-04 | 137.32ms | mfu 0.84%\n",
            "846 | loss 3.0245 | lr 4.230000e-04 | 138.88ms | mfu 0.84%\n",
            "847 | loss 2.9629 | lr 4.235000e-04 | 135.21ms | mfu 0.84%\n",
            "848 | loss 3.0468 | lr 4.240000e-04 | 134.67ms | mfu 0.84%\n",
            "849 | loss 2.9889 | lr 4.245000e-04 | 135.22ms | mfu 0.84%\n",
            "850 | loss 2.9117 | lr 4.250000e-04 | 136.96ms | mfu 0.83%\n",
            "851 | loss 2.9223 | lr 4.255000e-04 | 141.81ms | mfu 0.83%\n",
            "852 | loss 2.9944 | lr 4.260000e-04 | 134.60ms | mfu 0.83%\n",
            "853 | loss 2.9464 | lr 4.265000e-04 | 137.32ms | mfu 0.83%\n",
            "854 | loss 2.9684 | lr 4.270000e-04 | 137.13ms | mfu 0.83%\n",
            "855 | loss 2.9889 | lr 4.275000e-04 | 135.91ms | mfu 0.83%\n",
            "856 | loss 2.9292 | lr 4.280000e-04 | 140.97ms | mfu 0.82%\n",
            "857 | loss 3.0478 | lr 4.285000e-04 | 136.86ms | mfu 0.82%\n",
            "858 | loss 2.9908 | lr 4.290000e-04 | 137.77ms | mfu 0.82%\n",
            "859 | loss 2.8483 | lr 4.295000e-04 | 144.27ms | mfu 0.82%\n",
            "860 | loss 2.9238 | lr 4.300000e-04 | 142.85ms | mfu 0.81%\n",
            "861 | loss 3.0080 | lr 4.305000e-04 | 133.17ms | mfu 0.82%\n",
            "862 | loss 3.0630 | lr 4.310000e-04 | 137.82ms | mfu 0.82%\n",
            "863 | loss 2.9800 | lr 4.315000e-04 | 133.48ms | mfu 0.82%\n",
            "864 | loss 2.9214 | lr 4.320000e-04 | 140.05ms | mfu 0.82%\n",
            "865 | loss 2.9193 | lr 4.325000e-04 | 135.30ms | mfu 0.82%\n",
            "866 | loss 2.9458 | lr 4.330000e-04 | 147.25ms | mfu 0.81%\n",
            "867 | loss 2.8913 | lr 4.335000e-04 | 137.06ms | mfu 0.81%\n",
            "868 | loss 3.0075 | lr 4.340000e-04 | 137.37ms | mfu 0.81%\n",
            "869 | loss 3.0384 | lr 4.345000e-04 | 128.99ms | mfu 0.82%\n",
            "870 | loss 2.8889 | lr 4.350000e-04 | 131.09ms | mfu 0.82%\n",
            "871 | loss 2.8661 | lr 4.355000e-04 | 129.88ms | mfu 0.83%\n",
            "872 | loss 2.9868 | lr 4.360000e-04 | 129.25ms | mfu 0.83%\n",
            "873 | loss 2.8768 | lr 4.365000e-04 | 129.09ms | mfu 0.83%\n",
            "874 | loss 2.8949 | lr 4.370000e-04 | 129.63ms | mfu 0.84%\n",
            "875 | loss 2.8758 | lr 4.375000e-04 | 128.72ms | mfu 0.84%\n",
            "876 | loss 2.9525 | lr 4.380000e-04 | 128.60ms | mfu 0.84%\n",
            "877 | loss 2.8260 | lr 4.385000e-04 | 129.22ms | mfu 0.84%\n",
            "878 | loss 2.9638 | lr 4.390000e-04 | 128.99ms | mfu 0.85%\n",
            "879 | loss 2.9222 | lr 4.395000e-04 | 129.39ms | mfu 0.85%\n",
            "880 | loss 2.9621 | lr 4.400000e-04 | 129.82ms | mfu 0.85%\n",
            "881 | loss 2.9453 | lr 4.405000e-04 | 129.10ms | mfu 0.85%\n",
            "882 | loss 2.9562 | lr 4.410000e-04 | 133.48ms | mfu 0.85%\n",
            "883 | loss 2.9063 | lr 4.415000e-04 | 131.83ms | mfu 0.85%\n",
            "884 | loss 3.0321 | lr 4.420000e-04 | 128.42ms | mfu 0.85%\n",
            "885 | loss 2.8960 | lr 4.425000e-04 | 129.28ms | mfu 0.85%\n",
            "886 | loss 2.9677 | lr 4.430000e-04 | 129.41ms | mfu 0.85%\n",
            "887 | loss 2.9137 | lr 4.435000e-04 | 129.87ms | mfu 0.86%\n",
            "888 | loss 2.9769 | lr 4.440000e-04 | 130.42ms | mfu 0.86%\n",
            "889 | loss 2.8129 | lr 4.445000e-04 | 128.25ms | mfu 0.86%\n",
            "890 | loss 2.8605 | lr 4.450000e-04 | 129.81ms | mfu 0.86%\n",
            "891 | loss 2.8519 | lr 4.455000e-04 | 128.84ms | mfu 0.86%\n",
            "892 | loss 2.9401 | lr 4.460000e-04 | 129.40ms | mfu 0.86%\n",
            "893 | loss 2.8367 | lr 4.465000e-04 | 127.79ms | mfu 0.86%\n",
            "894 | loss 2.8883 | lr 4.470000e-04 | 131.19ms | mfu 0.86%\n",
            "895 | loss 2.8736 | lr 4.475000e-04 | 131.09ms | mfu 0.86%\n",
            "896 | loss 2.8116 | lr 4.480000e-04 | 128.83ms | mfu 0.86%\n",
            "897 | loss 2.7933 | lr 4.485000e-04 | 133.42ms | mfu 0.86%\n",
            "898 | loss 2.9070 | lr 4.490000e-04 | 127.90ms | mfu 0.86%\n",
            "899 | loss 2.9060 | lr 4.495000e-04 | 130.09ms | mfu 0.86%\n",
            "900 | loss 2.7927 | lr 4.500000e-04 | 128.09ms | mfu 0.86%\n",
            "901 | loss 2.7858 | lr 4.505000e-04 | 130.18ms | mfu 0.86%\n",
            "902 | loss 2.8917 | lr 4.510000e-04 | 131.43ms | mfu 0.86%\n",
            "903 | loss 2.8824 | lr 4.515000e-04 | 128.45ms | mfu 0.86%\n",
            "904 | loss 2.8773 | lr 4.520000e-04 | 131.04ms | mfu 0.86%\n",
            "905 | loss 2.7746 | lr 4.525000e-04 | 131.47ms | mfu 0.86%\n",
            "906 | loss 3.0296 | lr 4.530000e-04 | 129.56ms | mfu 0.86%\n",
            "907 | loss 2.9879 | lr 4.535000e-04 | 128.81ms | mfu 0.86%\n",
            "908 | loss 2.7863 | lr 4.540000e-04 | 129.47ms | mfu 0.86%\n",
            "909 | loss 2.8922 | lr 4.545000e-04 | 130.14ms | mfu 0.86%\n",
            "910 | loss 2.8780 | lr 4.550000e-04 | 129.15ms | mfu 0.86%\n",
            "911 | loss 2.8982 | lr 4.555000e-04 | 131.04ms | mfu 0.86%\n",
            "912 | loss 2.8031 | lr 4.560000e-04 | 129.80ms | mfu 0.86%\n",
            "913 | loss 2.9072 | lr 4.565000e-04 | 131.07ms | mfu 0.86%\n",
            "914 | loss 2.9120 | lr 4.570000e-04 | 128.81ms | mfu 0.86%\n",
            "915 | loss 2.8274 | lr 4.575000e-04 | 129.37ms | mfu 0.86%\n",
            "916 | loss 2.9874 | lr 4.580000e-04 | 129.57ms | mfu 0.86%\n",
            "917 | loss 2.8403 | lr 4.585000e-04 | 134.91ms | mfu 0.86%\n",
            "918 | loss 2.8153 | lr 4.590000e-04 | 129.68ms | mfu 0.86%\n",
            "919 | loss 2.8305 | lr 4.595000e-04 | 128.40ms | mfu 0.86%\n",
            "920 | loss 2.8632 | lr 4.600000e-04 | 129.30ms | mfu 0.86%\n",
            "921 | loss 2.8684 | lr 4.605000e-04 | 134.57ms | mfu 0.86%\n",
            "922 | loss 2.8844 | lr 4.610000e-04 | 128.87ms | mfu 0.86%\n",
            "923 | loss 2.8774 | lr 4.615000e-04 | 129.83ms | mfu 0.86%\n",
            "924 | loss 2.9593 | lr 4.620000e-04 | 129.54ms | mfu 0.86%\n",
            "925 | loss 2.8761 | lr 4.625000e-04 | 129.02ms | mfu 0.86%\n",
            "926 | loss 2.8140 | lr 4.630000e-04 | 130.84ms | mfu 0.86%\n",
            "927 | loss 2.7835 | lr 4.635000e-04 | 128.71ms | mfu 0.86%\n",
            "928 | loss 2.8236 | lr 4.640000e-04 | 129.02ms | mfu 0.86%\n",
            "929 | loss 2.9298 | lr 4.645000e-04 | 130.27ms | mfu 0.86%\n",
            "930 | loss 2.8754 | lr 4.650000e-04 | 128.71ms | mfu 0.86%\n",
            "931 | loss 2.7632 | lr 4.655000e-04 | 128.52ms | mfu 0.86%\n",
            "932 | loss 2.8632 | lr 4.660000e-04 | 129.95ms | mfu 0.86%\n",
            "933 | loss 2.9127 | lr 4.665000e-04 | 131.93ms | mfu 0.86%\n",
            "934 | loss 2.8034 | lr 4.670000e-04 | 129.03ms | mfu 0.86%\n",
            "935 | loss 2.9469 | lr 4.675000e-04 | 130.69ms | mfu 0.86%\n",
            "936 | loss 2.7399 | lr 4.680000e-04 | 134.20ms | mfu 0.86%\n",
            "937 | loss 2.8767 | lr 4.685000e-04 | 130.88ms | mfu 0.86%\n",
            "938 | loss 2.9751 | lr 4.690000e-04 | 128.67ms | mfu 0.86%\n",
            "939 | loss 2.8421 | lr 4.695000e-04 | 129.40ms | mfu 0.86%\n",
            "940 | loss 2.7962 | lr 4.700000e-04 | 128.97ms | mfu 0.86%\n",
            "941 | loss 2.8645 | lr 4.705000e-04 | 129.43ms | mfu 0.86%\n",
            "942 | loss 2.8065 | lr 4.710000e-04 | 130.66ms | mfu 0.86%\n",
            "943 | loss 2.8275 | lr 4.715000e-04 | 129.15ms | mfu 0.86%\n",
            "944 | loss 2.8656 | lr 4.720000e-04 | 132.25ms | mfu 0.86%\n",
            "945 | loss 2.8161 | lr 4.725000e-04 | 129.66ms | mfu 0.86%\n",
            "946 | loss 2.8504 | lr 4.730000e-04 | 140.73ms | mfu 0.85%\n",
            "947 | loss 2.9527 | lr 4.735000e-04 | 137.63ms | mfu 0.85%\n",
            "948 | loss 2.8691 | lr 4.740000e-04 | 136.70ms | mfu 0.85%\n",
            "949 | loss 2.6794 | lr 4.745000e-04 | 139.62ms | mfu 0.84%\n",
            "950 | loss 2.8221 | lr 4.750000e-04 | 135.91ms | mfu 0.84%\n",
            "951 | loss 2.8101 | lr 4.755000e-04 | 137.19ms | mfu 0.84%\n",
            "952 | loss 2.8631 | lr 4.760000e-04 | 141.48ms | mfu 0.83%\n",
            "953 | loss 2.7952 | lr 4.765000e-04 | 139.69ms | mfu 0.83%\n",
            "954 | loss 2.8900 | lr 4.770000e-04 | 136.55ms | mfu 0.83%\n",
            "955 | loss 2.9314 | lr 4.775000e-04 | 134.63ms | mfu 0.83%\n",
            "956 | loss 2.7403 | lr 4.780000e-04 | 133.79ms | mfu 0.83%\n",
            "957 | loss 2.8273 | lr 4.785000e-04 | 135.48ms | mfu 0.83%\n",
            "958 | loss 2.9234 | lr 4.790000e-04 | 140.95ms | mfu 0.83%\n",
            "959 | loss 2.7845 | lr 4.795000e-04 | 142.08ms | mfu 0.82%\n",
            "960 | loss 2.8454 | lr 4.800000e-04 | 136.60ms | mfu 0.82%\n",
            "961 | loss 2.8117 | lr 4.805000e-04 | 135.34ms | mfu 0.82%\n",
            "962 | loss 2.7965 | lr 4.810000e-04 | 143.18ms | mfu 0.82%\n",
            "963 | loss 2.7238 | lr 4.815000e-04 | 141.57ms | mfu 0.82%\n",
            "964 | loss 2.7803 | lr 4.820000e-04 | 138.57ms | mfu 0.81%\n",
            "965 | loss 2.7761 | lr 4.825000e-04 | 141.97ms | mfu 0.81%\n",
            "966 | loss 2.6831 | lr 4.830000e-04 | 140.39ms | mfu 0.81%\n",
            "967 | loss 2.8720 | lr 4.835000e-04 | 148.10ms | mfu 0.80%\n",
            "968 | loss 2.6508 | lr 4.840000e-04 | 140.33ms | mfu 0.80%\n",
            "969 | loss 2.7576 | lr 4.845000e-04 | 141.72ms | mfu 0.80%\n",
            "970 | loss 2.7373 | lr 4.850000e-04 | 144.90ms | mfu 0.80%\n",
            "971 | loss 2.7690 | lr 4.855000e-04 | 141.08ms | mfu 0.80%\n",
            "972 | loss 2.7704 | lr 4.860000e-04 | 142.28ms | mfu 0.80%\n",
            "973 | loss 2.8339 | lr 4.865000e-04 | 130.69ms | mfu 0.80%\n",
            "974 | loss 2.7488 | lr 4.870000e-04 | 131.56ms | mfu 0.81%\n",
            "975 | loss 2.8433 | lr 4.875000e-04 | 131.21ms | mfu 0.81%\n",
            "976 | loss 2.8460 | lr 4.880000e-04 | 129.45ms | mfu 0.82%\n",
            "977 | loss 2.8410 | lr 4.885000e-04 | 129.82ms | mfu 0.82%\n",
            "978 | loss 2.8140 | lr 4.890000e-04 | 130.93ms | mfu 0.83%\n",
            "979 | loss 2.8481 | lr 4.895000e-04 | 128.93ms | mfu 0.83%\n",
            "980 | loss 2.8355 | lr 4.900000e-04 | 128.54ms | mfu 0.83%\n",
            "981 | loss 2.7934 | lr 4.905000e-04 | 131.91ms | mfu 0.83%\n",
            "982 | loss 2.7515 | lr 4.910000e-04 | 133.70ms | mfu 0.84%\n",
            "983 | loss 2.8024 | lr 4.915000e-04 | 127.88ms | mfu 0.84%\n",
            "984 | loss 2.7986 | lr 4.920000e-04 | 129.13ms | mfu 0.84%\n",
            "985 | loss 2.8506 | lr 4.925000e-04 | 129.71ms | mfu 0.84%\n",
            "986 | loss 2.8432 | lr 4.930000e-04 | 129.80ms | mfu 0.85%\n",
            "987 | loss 2.7453 | lr 4.935000e-04 | 128.72ms | mfu 0.85%\n",
            "988 | loss 2.7175 | lr 4.940000e-04 | 130.15ms | mfu 0.85%\n",
            "989 | loss 2.7226 | lr 4.945000e-04 | 131.99ms | mfu 0.85%\n",
            "990 | loss 2.8527 | lr 4.950000e-04 | 130.29ms | mfu 0.85%\n",
            "991 | loss 2.7391 | lr 4.955000e-04 | 129.67ms | mfu 0.85%\n",
            "992 | loss 2.6757 | lr 4.960000e-04 | 128.94ms | mfu 0.85%\n",
            "993 | loss 2.7431 | lr 4.965000e-04 | 132.55ms | mfu 0.85%\n",
            "994 | loss 2.7747 | lr 4.970000e-04 | 128.69ms | mfu 0.85%\n",
            "995 | loss 2.7513 | lr 4.975000e-04 | 129.83ms | mfu 0.85%\n",
            "996 | loss 2.7914 | lr 4.980000e-04 | 134.77ms | mfu 0.85%\n",
            "997 | loss 2.7551 | lr 4.985000e-04 | 129.06ms | mfu 0.85%\n",
            "998 | loss 2.8347 | lr 4.990000e-04 | 132.21ms | mfu 0.85%\n",
            "999 | loss 2.7704 | lr 4.995000e-04 | 128.86ms | mfu 0.85%\n",
            "Created a PretokDataset with rng seed 42\n",
            "Created a PretokDataset with rng seed 42\n",
            "step 1000: train loss 2.7600, val loss 2.7572\n",
            "saving checkpoint to out\n",
            "wrote out/model.bin\n",
            "1000 | loss 2.7886 | lr 5.000000e-04 | 2595.44ms | mfu 0.77%\n",
            "1001 | loss 2.7945 | lr 0.000000e+00 | 131.34ms | mfu 0.78%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd llama2.c && gcc -O3 -o run run.c -lm"
      ],
      "metadata": {
        "id": "oUpbYitgNdMh"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cd ./llama2.c && ./run out/model.bin -z data/tok4096.bin -t 0.8 -n 256 -i \"Once upon a time \""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1-ylANKlOfCr",
        "outputId": "65e45080-2d6e-47bd-9f88-5c5cbd7d752f"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Once upon a time \n",
            "When they went to the garden, they went to the park to play. They saw a big, shiny hill and he wanted to play. They were very happy.\n",
            "Suddenly, the bird found a soft box. It looked around and saw a big tree. The bird said, \"I will help you open the window.\" \n",
            "He said, \"I don't want to play with you!\" But the bird was too busy. The bird was not sad anymore. The bird said, \"I have a game for you. We can see the bird.\" \n",
            "So, the bird saw the bird. The bird said, \"Wow! I'm a magic bird!\" The bird felt happy and thanked the bird. The bird and the little bird were very sad. The bird and the bird became good friends and had a fun time in the tree.\n",
            "achieved tok/s: 644.366197\n"
          ]
        }
      ]
    }
  ]
}