oai_batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663194\n",
      "{\"custom_id\": \"0{\\\"theme\\\": \\\"Dreams\\\", \\\"topic\\\": \\\"royal kingdoms\\\", \\\"style\\\": \\\"tragic\\\", \\\"feature\\\": \\\"a MacGuffin\\\", \\\"grammar\\\": \\\"anaphora\\\", \\\"num_paragraphs\\\": 1, \\\"expected_num_stories_in_completion\\\": 13}\", \"method\": \"POST\", \"url\": \"/v1/chat/completions\", \"body\": {\"model\": \"gpt-4o-mini\", \"messages\": [{\"role\": \"user\", \"content\": \"Write a short story (1 paragraphs) using very basic words that a preschool child could understand. \\nThe story should be about Dreams, include royal kingdoms, be tragic in its writing style and ideally feature a MacGuffin. The most important thing is to write an engaging easy story, but where it makes sense, demonstrate the use of anaphora. If you need to use proper names, make them from space-separated common words. Either don't give characters a name, or select from Mia, Alex, Jean, Samuel, Lily, Leo, Jose, Kim, Alice, Lena, Rita, Emmanuel, Anne, Peter, Maria or Luis. Complex story structure is great, but please remember to only use simple words.\"}], \"top_p\": 0.07}}\n"
     ]
    }
   ],
   "source": [
    "# 1. Create jsonl file with requests for batch completion\n",
    "\n",
    "import json\n",
    "import tiktoken\n",
    "import os\n",
    "from generate_stories import create_simple_story_prompt, iterate_params\n",
    "\n",
    "MODEL_PARAMETERS = {\"top_p\": 0.07}\n",
    "\n",
    "def get_batch_dataset(num_completions, model, offset=0):\n",
    "    \n",
    "    enc = tiktoken.encoding_for_model(model)\n",
    "    params_iterator = iterate_params()\n",
    "    for _ in range(offset):\n",
    "        next(params_iterator)\n",
    "    \n",
    "    while True:\n",
    "        lines = []\n",
    "        total_tokens = 0\n",
    "        for i in range(num_completions):\n",
    "            params = next(params_iterator)\n",
    "            prompt, num_stories_in_completion = create_simple_story_prompt(params.copy())\n",
    "            message_tokens = len(enc.encode(prompt))\n",
    "            total_tokens += message_tokens\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "\n",
    "            lines.append(json.dumps(\n",
    "                {\"custom_id\": str(i)+json.dumps(params | {\"expected_num_stories_in_completion\": num_stories_in_completion}),\n",
    "                \"method\": \"POST\",\n",
    "                \"url\": \"/v1/chat/completions\",\n",
    "                \"body\": {\"model\": model, \"messages\": messages, **MODEL_PARAMETERS}}\n",
    "                ))\n",
    "        \n",
    "        print(f\"Total input tokens: {total_tokens}\")\n",
    "\n",
    "        yield lines\n",
    "\n",
    "def write_batch_completion_file(num_completions, model, base_filename, offset=0):\n",
    "    iterator = get_batch_dataset(num_completions, model, offset)\n",
    "    counter = 1\n",
    "    filename = base_filename\n",
    "    while True:\n",
    "        lines = next(iterator)\n",
    "        filename = f\"{filename}_{str(counter)}.jsonl\"\n",
    "        with open(filename, \"w\") as fp:\n",
    "            fp.write(\"\\n\".join(lines))\n",
    "        counter += 1\n",
    "        yield filename\n",
    "    \n",
    "if not os.path.exists(\"data\"):\n",
    "    os.makedirs(\"data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:   0%|          | 0/100000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663233\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  10%|█         | 10000/100000 [49:24<7:24:37,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663038\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  20%|██        | 20000/100000 [1:37:17<6:28:06,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663496\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  30%|███       | 30000/100000 [2:26:42<5:42:27,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663234\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  40%|████      | 40000/100000 [3:31:14<5:30:30,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663117\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  50%|█████     | 50000/100000 [4:17:09<4:18:54,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663300\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  60%|██████    | 60000/100000 [4:59:29<3:14:16,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663337\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  70%|███████   | 70000/100000 [5:48:54<2:26:32,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663055\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  80%|████████  | 80000/100000 [6:37:47<1:37:42,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663122\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation:  90%|█████████ | 90000/100000 [7:21:42<47:18,  3.52it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1663111\n",
      "Batch status: validating\n",
      "The input file is being validated. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: in_progress\n",
      "The batch is currently being processed. Please wait...\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: finalizing\n",
      "The batch is completed and the results are being prepared.\n",
      "Batch status: completed\n",
      "The batch is complete, downloading the results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch Generation: 100%|██████████| 100000/100000 [8:07:07<00:00,  3.42it/s]\n"
     ]
    }
   ],
   "source": [
    "# 2. Execute Batch Jobs\n",
    "\n",
    "import os\n",
    "import time\n",
    "from datetime import datetime\n",
    "from openai import OpenAI\n",
    "from tqdm import tqdm\n",
    "        \n",
    "NUM_COMPLETIONS = 100_000\n",
    "NUM_COMPLETIONS_PER_REQUEST = 10_000 # Calculate this based on rate limits, to be checked at https://platform.openai.com/settings/organization/limits\n",
    "MODEL = \"gpt-4o-mini\"\n",
    "MAX_RETRIES = 3\n",
    "\n",
    "client = OpenAI(api_key=os.environ[\"OPENAI_API_KEY_SIMPLESTORIES\"])\n",
    "\n",
    "def check_batch_status(batch_id, batch_number, directory):\n",
    "    while True:\n",
    "        batch_status = client.batches.retrieve(batch_id)\n",
    "\n",
    "        status = batch_status.status\n",
    "        print(f\"Batch status: {status}\")\n",
    "\n",
    "        if status == \"validating\":\n",
    "            print(\"The input file is being validated. Please wait...\")\n",
    "        elif status == \"failed\":\n",
    "            print(\"The input file has failed validation.\")\n",
    "            return False\n",
    "        elif status == \"in_progress\":\n",
    "            print(\"The batch is currently being processed. Please wait...\")\n",
    "        elif status == \"finalizing\":\n",
    "            print(\"The batch is completed and the results are being prepared.\")\n",
    "        elif status == \"completed\":\n",
    "            print(\"The batch is complete, downloading the results...\")\n",
    "            download_batch_results(batch_status.output_file_id, batch_number, directory)\n",
    "            return True\n",
    "        elif status == \"expired\":\n",
    "            print(\"The batch was not completed within the 24-hour time window.\")\n",
    "            return False\n",
    "        elif status == \"cancelling\":\n",
    "            print(\"The batch is being cancelled. Please wait...\")\n",
    "        elif status == \"cancelled\":\n",
    "            print(\"The batch was cancelled.\")\n",
    "            return False\n",
    "        else:\n",
    "            print(\"Unknown status encountered.\")\n",
    "\n",
    "        time.sleep(30)  # Wait for 30 seconds before checking the status again\n",
    "        \n",
    "def download_batch_results(output_file_id, batch_number, directory):\n",
    "    with open(os.path.join(directory, \"output_file_ids.txt\"), \"a\") as f:\n",
    "        f.write(output_file_id + \"\\n\")\n",
    "\n",
    "    file_response = client.files.content(output_file_id)\n",
    "    \n",
    "    filename = f\"{directory}/batch_data_{batch_number}.jsonl\"\n",
    "    with open(filename, 'w') as f:\n",
    "        f.write(file_response.text)\n",
    "total_completions = 0\n",
    "batch_number = 0\n",
    "consecutive_failures = 0\n",
    "\n",
    "directory = os.path.join(\"data\", f\"batches_{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}\")\n",
    "os.makedirs(directory, exist_ok=True)\n",
    "os.makedirs(os.path.join(directory, \"prompts\"), exist_ok=True)\n",
    "\n",
    "base_filename = os.path.join(directory, \"prompts\", \"batch\")\n",
    "batch_writer_iter = write_batch_completion_file(NUM_COMPLETIONS_PER_REQUEST, MODEL, base_filename, offset=10_000)\n",
    "with tqdm(total=NUM_COMPLETIONS, desc=\"Batch Generation\") as pbar:\n",
    "    while total_completions < NUM_COMPLETIONS and consecutive_failures < MAX_RETRIES:\n",
    "        try:\n",
    "            # 1. Write the batch completion file\n",
    "            batch_number += 1\n",
    "            filename = next(batch_writer_iter)\n",
    "\n",
    "            # 2. Upload the batch file\n",
    "            batch_input_file = client.files.create(\n",
    "                file=open(filename, \"rb\"),\n",
    "                purpose=\"batch\"\n",
    "            )\n",
    "            batch_input_file_id = batch_input_file.id\n",
    "            with open(os.path.join(directory, \"input_file_ids.txt\"), \"a\") as f:\n",
    "                f.write(batch_input_file_id + \"\\n\")\n",
    "\n",
    "            # 3. Create the batch job\n",
    "            batch_info = client.batches.create(\n",
    "                input_file_id=batch_input_file_id,\n",
    "                endpoint=\"/v1/chat/completions\",\n",
    "                completion_window=\"24h\",\n",
    "                metadata={\n",
    "                    \"description\": f\"Simple Stories Story Generation - batch {batch_number}, n={NUM_COMPLETIONS_PER_REQUEST}\"\n",
    "                }\n",
    "            )\n",
    "\n",
    "            batch_id = batch_info.id\n",
    "            with open(os.path.join(directory, \"batch_job_ids.txt\"), \"a\") as f:\n",
    "                f.write(batch_id + \"\\n\")\n",
    "\n",
    "            # 4. Check the status and download the results\n",
    "            if check_batch_status(batch_id, batch_number, directory):\n",
    "                total_completions += NUM_COMPLETIONS_PER_REQUEST\n",
    "                pbar.update(NUM_COMPLETIONS_PER_REQUEST)\n",
    "                consecutive_failures = 0\n",
    "            else:\n",
    "                consecutive_failures += 1\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {e}\")\n",
    "            consecutive_failures += 1\n",
    "\n",
    "    if consecutive_failures >= MAX_RETRIES:\n",
    "        print(f\"Stopping due to {MAX_RETRIES} consecutive failures.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Format batch output\n",
    "import re\n",
    "import json\n",
    "import os\n",
    "\n",
    "from generate_stories import process_completion\n",
    "\n",
    "def format_jsonl(input_files, output_file, input_files_nucleus=[]):\n",
    "    assert not os.path.isfile(output_file), \"output file already exists\"\n",
    "\n",
    "    for k, input_file in enumerate(input_files_nucleus + input_files):\n",
    "        with open(input_file, 'r') as infile, open(output_file, 'a') as outfile:\n",
    "            for line in infile:\n",
    "                data = json.loads(line)\n",
    "                \n",
    "                custom_id = data['custom_id']\n",
    "                match = re.search(r'{.*}', custom_id)\n",
    "                if match:\n",
    "                    params = json.loads(match.group(0))\n",
    "                else:\n",
    "                    continue\n",
    "                \n",
    "                completion = data['response']['body']['choices'][0]['message']['content']\n",
    "                gen_model = data['response']['body']['model']\n",
    "                \n",
    "                json_struct = process_completion(gen_model, completion, params, expected_num_stories=params.get(\"expected_num_in_stories\", None))\n",
    "                json_struct = [item | {\"nucleus\":  k >= len(input_files_nucleus)} for item in json_struct]\n",
    "                lines = [json.dumps(item) for item in json_struct if 'story' in item]\n",
    "                outfile.write(\"\\n\".join(lines) + \"\\n\")\n",
    "\n",
    "input_dir = os.path.join('data', 'batches_2024-09-03-11-24-05')\n",
    "input_files = []\n",
    "input_files.extend([os.path.join(input_dir,file) for file in os.listdir(input_dir) if file.endswith('.jsonl') and os.path.isfile(os.path.join(input_dir, file))])\n",
    "input_files.append(r'data\\batches_2024-09-02-19-54-39\\batch_data_1.jsonl')\n",
    "output_file = os.path.join(input_dir, 'processed.jsonl')\n",
    "\n",
    "format_jsonl(input_files, output_file, input_files_nucleus=[r\"data\\batches_2024-09-02-16-47-53\\batch_data_1.jsonl\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File successfully converted to data\\batches_2024-09-03-11-24-05\\processed.parquetl\n"
     ]
    }
   ],
   "source": [
    "# 3.1: Optionally convert to parquet\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "parquet_file_path = output_file.replace(\".jsonl\", \".parquet\")\n",
    "\n",
    "df = pd.read_json(output_file, lines=True)\n",
    "df.to_parquet(parquet_file_path, engine='pyarrow', compression='snappy')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4. Proceed to either analyse_dataset.ipynb or embeddings.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}