notebooks/training_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "16096d14-64b1-47db-8408-e3b800d6549e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"Atomi/semeval_2013_task_7_beetle_5way\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "371ccd3e-37c0-423b-9c8b-9f03287e92bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dspy\n",
    "import openai\n",
    "\n",
    "gpt3_turbo_instruct = dspy.OpenAI(model=\"gpt-3.5-turbo-instruct\", api_key=\"key\", max_tokens=1000)\n",
    "llama3_ollama = dspy.OllamaLocal(model=\"llama3\", max_tokens=1000)\n",
    "# lm_llama3_HF = dspy.HFModel(model=\"meta-llama/Meta-Llama-3-8B-Instruct\")\n",
    "# https://dspy-docs.vercel.app/api/local_language_model_clients/HFModel\n",
    "\n",
    "dspy.settings.configure(lm=llama3_ollama)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c02deddb-4e59-4fed-8ff5-eb5012133072",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question_id': 'HYBRID_BURNED_OUT_EXPLAIN_Q1',\n",
       " 'question': 'Explain your reasoning.',\n",
       " 'question_qtype': 'Q_EXPLAIN_SPECIFIC',\n",
       " 'question_module': 'SwitchesBulbsParallel',\n",
       " 'question_stype': 'PREDICT',\n",
       " 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery',\n",
       " 'reference_answer_quality': 'BEST',\n",
       " 'student_answer': 'bulb b and c will not be in a closed path with the battery',\n",
       " 'label_5way': 'correct',\n",
       " 'test_set': 'training'}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "57197bab-f09a-43a4-aaa8-d8f8e71a453d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AssessmentSignature(dspy.Signature):\n",
    "    \"\"\"\n",
    "    Role: Academic Tutor\n",
    "    Task: Make an assessment of the student's answer by comparing it with the reference answer.\n",
    "    You are essentially assessing whether the student's answer was correct, or needs additional feedback.\n",
    "    \n",
    "    The value for assessment must be EXACTLY one of the following classes:\n",
    "    \"correct\", \"partially_correct_incomplete\", \"contradictory\", \"non_domain\", \"irrelevant\"\n",
    "    Include the underscores if the class contains it.\n",
    "\n",
    "    Definitions of the classes:\n",
    "    - correct: if the student answer is a complete and correct paraphrase of the reference answer\n",
    "    - partially_correct_incomplete: if it is a partially correct answer containing some but not\n",
    "    all information from the reference answer\n",
    "    - contradictory: if the student answer explicitly contradicts the reference answer\n",
    "    - irrelevant:  if the student answer is talking about domain content but not providing the necessary information\n",
    "    - non_domain: if the student utterance does not include domain content, e.g., “I don’t know”, “what the book says”, “you are stupid”\n",
    "\n",
    "    The \"question\" field doesn't provide too much context apart from the final few words of the problem.\n",
    "    So don't focus on the question too much.\n",
    "    \"\"\"\n",
    "\n",
    "    question: str = dspy.InputField()\n",
    "    student_answer: str = dspy.InputField()\n",
    "    reference_answer: str = dspy.InputField()\n",
    "\n",
    "    assessment: str = dspy.OutputField(desc=\"the final assessment\")\n",
    "\n",
    "class ASAGCoT(dspy.Module):\n",
    "    \"\"\"Assess the student's answer to the question by comparing with the reference answer\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.prog = dspy.ChainOfThought(AssessmentSignature)\n",
    "\n",
    "    def forward(self, question: str, student_answer: str, reference_answer: str):\n",
    "        output = self.prog(question=question, student_answer=student_answer, reference_answer=reference_answer)\n",
    "        \n",
    "        output.assessment = str(output.assessment).lower()\n",
    "        dspy.Suggest(\n",
    "            output.assessment in [\"correct\", \"partially_correct_incomplete\", \"contradictory\", \"non_domain\", \"irrelevant\"],\n",
    "            f\"'{output.assessment}' must be exactly one of 'correct', 'partially_correct_incomplete', 'contradictory', 'non_domain', 'irrelevant'\"\n",
    "        )\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e93838a6-45f2-4f77-85bd-001a777774ca",
   "metadata": {},
   "source": [
    "## Trying Zero shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "32b285e4-c131-45f2-b0ce-c743117cf2eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "with dspy.context(lm=gpt3_turbo_instruct):\n",
    "    asag_cot_pipeline = ASAGCoT()\n",
    "\n",
    "question, s_answer, r_answer = dataset[\"train\"][0][\"question\"], dataset[\"train\"][0][\"student_answer\"], dataset[\"train\"][0][\"reference_answer\"]\n",
    "assessment = asag_cot_pipeline.forward(question, s_answer, r_answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "1625642d-47a6-4ad8-89a4-aa36f49e3ed6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Explain your reasoning.'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "1bf42f5d-162a-4af4-9351-818e8d19ba78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bulb b and c will not be in a closed path with the battery'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "2bb804cb-a0c1-4efd-a51c-36f1d898efbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'If bulb A burns out, B and C are no longer in a closed path with the battery'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "1d360c62-97d3-40e2-a543-ad21cecadf01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Prediction(\n",
       "    rationale=\"Here is the assessment:\\n\\n---\\nQuestion: Explain your reasoning.\\nStudent Answer: bulb b and c will not be in a closed path with the battery\\nReference Answer: If bulb A burns out, B and C are no longer in a closed path with the battery\\nReasoning: Let's think step by step in order to assess whether the student's answer is correct or needs additional feedback. The student's answer states that bulbs B and C will not be in a closed path with the battery when bulb A burns out, which is partially correct but incomplete. The reference answer explicitly states the condition under which bulbs B and C are no longer in a closed path with the battery, which is if bulb A burns out. Therefore, the student's answer is partially correct but incomplete.\",\n",
       "    assessment='partially_correct_incomplete'\n",
       ")"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assessment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "cc3238d1-91fe-4cbe-bc16-41546dcf4112",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'correct'"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"][0][\"label_5way\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "6ed264fe-be0c-482f-9fd5-5a89e395e37c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt3_turbo_instruct.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4821839-081d-4976-903c-e6f26e89a950",
   "metadata": {},
   "source": [
    "## Creating a Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "9afaa0d1-3c03-4838-b52a-502f8383ad02",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = dataset[\"test\"]\n",
    "\n",
    "dspy_test_dataset = [\n",
    "    dspy.Example(\n",
    "        question=data[\"question\"],\n",
    "        student_answer=data[\"student_answer\"],\n",
    "        reference_answer=data[\"reference_answer\"],\n",
    "        assessment=data[\"label_5way\"]\n",
    "    ).with_inputs(\"question\",\"student_answer\",\"reference_answer\")\n",
    "    for data in test_set\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "2efa7596-c957-4290-94a9-95231ee8d464",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we have 1489 test data points in total\n"
     ]
    }
   ],
   "source": [
    "print(f\"we have {len(dspy_test_dataset)} test data points in total\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "754ecaf1-9c03-459b-a27c-ad96076051d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = dataset[\"train\"]\n",
    "\n",
    "best_quality_dataset = [example for example in train_set if example[\"reference_answer_quality\"] == \"BEST\"]\n",
    "\n",
    "dspy_dataset_best_quality_all = [\n",
    "    dspy.Example(\n",
    "        question=data[\"question\"],\n",
    "        student_answer=data[\"student_answer\"],\n",
    "        reference_answer=data[\"reference_answer\"],\n",
    "        assessment=data[\"label_5way\"]\n",
    "    ).with_inputs(\"question\",\"student_answer\",\"reference_answer\")\n",
    "    for data in best_quality_dataset\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "55f17946-11a0-4ca9-906a-8f66226552e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c will not be in a closed path with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no closed path that contains the battery, bulb b, and bulb c', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'creates a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are all in the same path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c will no longer be in closed paths with the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'damaged bulb a creates a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a creates a gap in teh circuit.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the bulb creates a gap in the closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a creates a gap in teh path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap in the path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'seperates the voltage', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs B and C will be on open paths with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a is in the same path of bulbs b and c', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because there is a gap.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are all on the same path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'i dont know then', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'njo', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'nooyutgyhjiko', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs b and c will not be in a closed path with the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are still contained in a closed path with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'l', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The path is not closed.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'a is in the same path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a still has a path to the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'connection', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a is contained in their closed paths', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a is contained in the same closed path as both bulbs b and c', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a is in the closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a creates a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and are not in a closed path with bulb a', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A is on the same path as B and C', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A and B and C are on the same paths', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are not on a closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'both bulbs B and C are in the same path as bulb A', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'tell answer', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A will create a gap in the path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb a created a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b will not be in a closed path with the battery. bulb c will not be in a closed path with the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb A is damaged the closed path has a gap. bulb b and bulb c are no longer on a closed path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb a is damaged bulb b and bulb c are no longer in a closed path with the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap a bulb A', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'damaging bulb a creates a gap.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is not a closed path that contains the batteyr and bulb b and bulb c', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are still contained in closed paths with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are in a closed path with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b is in a closed path with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B and C will no be in a closed path with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A is contained in closed paths with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are still contained in closed paths with the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c have separate paths within the closed circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'both bulbs B and C each have their own separate path to the battery within the closed circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'damaged bulb a creates a gap in the path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are in separate paths.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There is a gap in the path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are conected to each other', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'neither bulb b or bulb c will be contained on a closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are no longer on a closed cirucit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Without A, there\"s no closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'B and C will no longer be contained in a closed path to the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because they are in series with a', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A creates a gap in the path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there would be a gap in the closed circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Damaged bulb A will create a gap.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'still connected to the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in the same path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Damaged bulb A creates a gap in the circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because the damaged bulb a creates a gap in the path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B, and C are still ina contained path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'seperate circuits', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'contained in closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'A GAP WIL BE CREATED', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A is not in the same path as bulbs B and C.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A will no longer be completing the circuts.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb A creates a gap in the closed circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is a gap caused by bulb A', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because it would cause a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the damaged bulb a creates a gap in the path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because it damages the complete circuit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'it is on a closed path', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There is a gap in the closed path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'it creats a gap so there are no closed paths', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Seperate paths connected to the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Damaging Bulb A would disrupt the closed path with the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb A burns out then bulbs B and C will no longer be in a closed path.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'B nor C will have a contained closed paths to the battery', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs B and C are still contained in a closed path to the battery.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'that creates a gap', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because there is a difference in electrial states between the two terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is a difference between the electrical states of the two battery  terminals because they are not connected to each other', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the cheminal reaction of the volatge is there wther or not there is an incomplete circuit', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery will always have a voltage.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery is a chemical reaction that does not rely on the circuit being complete or not.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminals in the battery will have an electrical state and chemical reaction that will continuously flow.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery is in a constant state of electrical charge', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a battery is in a constant state of electricity', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery always has a charge in its positive terminal', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the terminals are not connected', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the positive terminal and negative terminal are not connected', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the positive battery terminal is not connected to the negative battery terminal.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The chemical reation still exists', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery uses the chemical reation to create voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is no short circuit to the battery.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery still holds the electrical charge if there is an incomplete circuit.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the gap will not effect the voltage.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because the chemical reaction is not effected by a gap.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminals must have a difference in electrical states to have voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the difference in the electrical states between two terminals causes a chemical reaction', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'There is always a difference in voltage between two battery terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The gap causes a difference in electrical states.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery terminals have different electrical states.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the voltage doesnt change in the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because of the chemical in the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because of the chemical reaction in the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because they are in the same electrical state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'and they are in the same state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'l', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'There is a difference in electrical states between two terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'It is different from the battery.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'difference in electrical state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'gap in terminal', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the voltage is the difference in electrical states between two terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminal has an electrical state between terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a chemical reaction between terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the battery has electricity and the gap does not', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery always has voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminals are not connected', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is a gap between terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a battery uses voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'dont know', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because there is no gap between the positive and negative terminal', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminals used by the battery have no gap.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it is the difference of the electrical state between the two battery terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'It is the difference of the electrical states caused by a chemical reaction between the two terminals of the battery.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the electrical state difference between the battery\"s two terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'i dont know', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because it has a positive terminal', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'tell answer', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery will have a different electrical state than the circuit', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The difference in electrical state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery has a different electrical state than the incomplete circuit', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery uses the electrical state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Battery will always have voltage.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery keeps a chemical reaction reaction.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery uses a chemical reaction to maintain the voltage,', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is always voltage even if there is a gap', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is a chemical reaction between two terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a terminal has an electrical state where theres a chemical reaction', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery always has voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery has a chemical reaction that causes a voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a batter uses a chemical reaction to create voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage of the battery depends on the chemical interaction inside the battery. It does not depend on components in a curcuit.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage of a battery does not depend on the components in a curcuit.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage of the battery does not depend on the components in a curcuit. It depends on a chemical reaction.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'becasuse it is electrically charged', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'yes, because there is a chemical reaction', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'There is a chemical reaction.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'circuit do not effect electrical state between terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'circuits do not effect the electrical state between two terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'an incomplete circuit would not effect the the electrical state between the terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminal\"s electrical state is not effected by the incomplete circuit.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'There is still a voltage is the different terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The terminal is still connected to an electrical state of the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'it always has the same voltage due to a chemical reaction inside of it', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a battery is always in the same electrical state. it has a chemical reaction inside of it.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the battery has its own electrical state within given by the lead', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery has its own chemical reaction', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because terminals are seperate', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'not connected', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'battery uses charge', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a batter will always have a voltage even if it is 0', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery will always have a voltage even if it is 0', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery always has a chemical reaction, positive or negative', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a chemical reaction inside the battery keeps the positive and negative terminals at different electrical states at all times.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'A battery will always have a voltage because the electrical states of the positive and negative terminals are always different.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'A chemical reaction inside the battery makes the electrical states of the positive and negative terminals different at all times.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The two terminals of a battery will always have different electrical states due to a chemical reaction inside the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The voltage does not change.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it will still be connected to one of the terminals', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the gap creates a difference in electrical states', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage of a battery is the result of chemical reactions within the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it has one regardless of the circuit', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'battery always has voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because there is a difference between the electrical states of the two terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because there is still a difference in electrical states.  One side could have a positive connection and the other could have a negative connection', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because the terminals are not in a closed path, so you are giving the voltage of the battery itself', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the battery always has a voltage unless it\"s dead', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it has two electrial states within itself.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because they aren\"t connected', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'battery will always have a voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'There will be a difference in measured electrical states', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a gap in the circuit does not affect the voltage of the battery.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'An incomplete circuit does not affect voltage.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'it can be 0', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'a battery always has voltage unless it is damaged', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'circuit has nothing to do with the batteries voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Well, it will have a voltage of zero, if that\"s what you are asking.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The battery\"s voltage comes from a chemical reaction that takes place within the battery itself.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the electrical states between the two battery terminals will be different no matter what', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'there is a gap', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'beacause the two terminals still have an electrical state', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the battery always has a voltage because there is a positive terminal and there is a negative terminal.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it is measuring the difference in electrical states between two terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because the battery contains chemicals that differe in electrical states.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage stays the same within the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'I don\"t know', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because the two terminals will always have different electrical states.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because voltage is the difference in electrical states between two terminals.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the battery has its own chemical reacion between the positive and negative tterminals it contains', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'is the difference between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'is the difference between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the is a difference in the terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'is the difference in electrial stat between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electrical states between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in the electrical states of the positive and negative terminals of a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement between the electrical states of the positive and negative terminals of a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'electricity difference between the battey\"s positive and negative terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a measurement of current and flowing the a source.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a measurement of current flowing through a source.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a difference between the electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between the positive and negative terminals of a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference of electrical states in a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electricity between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'teh difference of electrical states between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Two terminals that have different electrical states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a measurement of electricity', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the different in the positive terminal and negative terminal.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between the electrical state of the positive terminal and the negative terminal.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the measure of electrical charges in a component.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the amount of electrical charge within a component.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference between tow terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement of power of electricity', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in connections between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electrical state between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference in electrical states between two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in batteries', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference between the baterry and termnial', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in the electrical state between the battery and terminal', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference between states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'it is the difference between the electrical states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference in the electrical states between the terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference in electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The measurement in difference of the electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'measurement of the difference in electrical states between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurment of power to a source of energy', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the power of the terminal', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'gods state', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between electrical charges of battery terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference of chemical signals between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between electrical charges in a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electricity created by chemical reactions between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference between the two ends of the battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'different chemical output', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Difference between the electrical state of the two ends of the battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'potential difference', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference in eletricity', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'potential difference between 2 terminal', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference in electrical state between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between the positive and negative measurements of power', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference in electrical charge between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Voltage is the difference between the positive terminal and negative terminal in a battery.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Voltage is the difference in electrical states between two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the amount of volts', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the distance between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a difference between two terminals with a measurement', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a difference is electrical states between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference in electrical states between two terminals of a circuit', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electrical states of two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference betweej two terminals that are not connected', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference between two terminals that produces and electrical state', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the diffrence between states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the diffrence in electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'volatge is the diffrence in electrical state between two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a measure of electric power', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'a measure of the strength of electricity', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in size between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement of the difference between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'i dont know', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': '...', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement of the difference between two terminals in an electric state', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'distance between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'energy between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference between paths of terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'negative and positive difference', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the amount of power a battery emits', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in power a battery emits', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in an electrical state a battery has', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between two different terminal\"s electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is a reading of the difference in electrical states between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Voltage is the charge given when two terminals are connected.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference between the terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': '0.0', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference in the electrical states between two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference in electrical states of terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': '0', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The measure of electritcy across a power source', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': '0.0 volts', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'measure of electicity', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'difference between two electrical currents', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Voltage is the difference in electrical states between a bulb terminal and a battery terminal.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the amount of electricity in a batery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The amount of charge.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'electrucal charge', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the differance between the 2 terminals of a batter\"s state', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'measure of electrical current?', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Voltage is how much energy is going on inside a battery.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference is electrical states between two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in charges of two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference in electrical states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference between positive and negative', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between 2 terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is a property of batteries. voltage is the difference between electrical currents in two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The measured difference between two electrical states.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference in charge between the two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement of the electrical current that occurs when the chemicals are mixed together.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The amount of electric being put out', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The electrical difference between the two terminals of a battery', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the difference between terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference in the states of the terminal', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The measure between 2 different terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'measure of the difference between the elecrical states of two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it causes an open circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it causes the path to be open', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it opens the path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the electricity will not flow through the wires if the circuit path is incomplete.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The bulbs will not be powered if the circuit path is incomplete.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The open switch makes the circuit incomplete.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it creates an open path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it causes a short circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it blocks the charge of the circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because the the circuit is complete.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it creates a gap in the current.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it creates a gap int he circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it creates a gap in the circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because it creates a gap, which opens a path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'i have no idea', 'reference_answer': 'the open switch creates a gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'a switch creates power', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'a switch creates a closed circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'a switch creates a closed path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It breaks the closed path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'A switch creates a break in the closed path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'A switch creates a gap in the path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because the path is short', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'makes  a short circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because there is no closed path to the battery', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because there is no closed path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It prevents the path from being closed.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It is not a closed path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'There is a gap in the circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'makes it an open path rather then closed', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it closes the pathway', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'effects the connection from battery to bulb', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'creates a connection', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'creates a circuit?', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the electricity can\"t pass through', 'reference_answer': 'the open switch creates a gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'there is a gap', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because all the terminals are connected to other terminals, an open switch disrupts that.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because an open switch creates a gap.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It turns off all the bulbs.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It creates a gap in the circut.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the path is not closed anymore', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it does\"nt make the circuit complete anymore', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the paths of the circuit are no longer complete', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the path is complete', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The open switch makes the path open', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It stops the flow of electricity.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It makes the circuit incomplete.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the switch opens the closed circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the switch opens the closed path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the voltage does not reach all the components', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the gap in the circuit causes the circuit to be an open circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It makes the circuit an incomplete path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the circuit is no longer complete when the switches are open', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because it opens the connection', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because it stops the battery from being connected.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It creates a short circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'an open switch causes a gap.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The path is not closed.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'an open switch incompletes the circut', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'an open switch creates an incomplete circut', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it creates a gap which in turn created an open path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'open switch creates a gap in the path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'brings he flow of electricity', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'charges the bulb', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'switch creates circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because it decides whether the bulb will be lit or not', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because if the gap is open the bulb will not go on', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it creates a gap, thus the circuit becomes incomplete', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The are connected to the same closed path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the voltage does not reach the bulbs.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It cuts the current from the battery', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the current cannot pass an open switch', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the bulb is not longer in a closed circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'There must be a closed path, and an open switch makes it not closed.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the path between the battery and the light bulb is not closed', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it opens the circuit, therefore the circuit is not closed', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'And open switch breaks the closed path, which is essential to function properly', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it breaks the closed circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'An open switch means the circuit is not a closed path.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it cuts the flow of electricity', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it opens the closed path of the battery', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the battery runs in a closed circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it opens the circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'any open swith does no allow the bulbs to turn on', 'reference_answer': 'the open switch creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'the currunt is blocked off', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It creates a gap.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'If a switch is open, the circut is no longer closed.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'it creates a gap in the circuit', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'An open switch stops the energy flow', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'When a switch is open the circuit is no longer complete', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because they are in a series, because a closed circuit acts together.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because it stops the flow of one side of a battery from reaching the other light bulbs.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'An open switch interrupts the flow of the current.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'It breaks the closed pathway that the bulbs require to work.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'There is no closed path', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the current can not reach the bulb if the circuit is open.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Then the bulb and battery will not be in a contained circuit.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'An open switch impacts a circuit because it causes the path to be closed, therefore the connection cannot be made.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'because it changes the path from closed to open', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are contained in the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B  and Bulb C are contained in the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'contained within the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are contained within a closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are contained in the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are contained on the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are contained in the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B and bulb C are contained in the same closed path.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are contained in the same closed path.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same pah', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because bulbs B and C are contained in the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are in the same closed path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and bulb C are contained in different path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'different paths with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and bulb C are contained in different paths with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because they are in the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and Bulb C are contained in the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained in the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained in the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are contained in the same path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and C are contained in the same closed paths with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because bulb b andc are in the same closed path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because bulb b and c are in the same closed path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs B and C are on the same path as the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs B and C are on the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are contained in the same closed path with the same battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'both bulbs are on the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are contained in the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs B and C are contained in the same closed path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are contain in the same closed path.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are contained in the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and bulb c are on the same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and bulb C are contained within the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are on the same paths', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and Bulb C are contained in the same pathway', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained in the same path to the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because they share the same path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs B and C are contained in the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B and bulb C are contained in the same path as the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'same path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'same closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained in the same path with the battery and are affected by one anotehr', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained in the same path as the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb B and Bulb C are in the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same ppath', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'yes', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are both within the same path.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are both on the same closed path as the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are wired in the same path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'same path to the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb b and c are in a closed path', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'both are on same closed paths', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are on the same pathway as the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are both in the same closed path and effect each other', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained on the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained inthe same paths with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs B and C are contained on the same path with the battery.', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained in the same closed path with the battery', 'reference_answer': 'B and C are in the same path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they are connected to opposite terminals', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'beacuse terminal 1 is connected to the negative terminal and the battery is connected at the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is a gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i do not know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it is not connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is no gap between terminal 1 and the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 is connected to the positive terminal with no gaps', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'It was a mistake. I looked at it wrong.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'We switched the terminals around.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There was a gap in b. The measurement between the positive battery terminal and terminal 1 was 1.5v.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The path is not closed. The distance between the positive battery terminal and terminal a is 1.5v.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive terminal of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there was no gap from the positive terminal of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because 1.5 v minus 0 v equals 1.5 v', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage is the difference between the electrical state of the terminals.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative terminal of the battery and has 0 v, and the positive terminal of the battery has 1.5 v.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the Meter read the difference in volts between the positive battery terminal and terminal 1 which has a negative charge', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Meter read the difference in volts between the positive battery terminal and terminal 1 which is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The gap causes terminal 1 to have a negative battery characteristic which is different compared  to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Bulb terminals 3 and 4', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i don\"t know.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between the circuit.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between terminal 1 and the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between terminal 1 and the negative battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because there was no gap.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 had no gap.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 is seperated from the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they were connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is a gap between terminal 3 and 4', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there wasnt a gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage doesnt change', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the gap is open', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the paths are seperated', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the terminal was connected to the positive battery', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the terminals were connected to the positive terminals of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap between the terminals', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 is connected to the negative terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dont know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was a gap in bulb B', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'circuit was complete until gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'gap, battery and electricity', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'they are in a closed circuit', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i don\"t know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a gap in the path', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap in the path', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a gap in the circuit', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'don\"t know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gapp between the positive terminal and terminal 1.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap between terminal 1 and the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'It was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 was not connected to the negative terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'fjdajf', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal one is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'whatever', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The positive terminal has 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap between the postive terminal and terminal 1', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative end on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative terminal on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal is connected to the bulb', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because 1.5 v is the difference between the positive terminal and terminal 1', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal is connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 is connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The positive terminal and terminal 1 read 1.5 V because the difference across the terminals is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage between terminal 1 and the positive terminal is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative terminal so the voltage reading is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': '?', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I  do not know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap after bulb B.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap after bulb b', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'All components are in a closed circuit.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the 1.5 volt battery.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dont know.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 and the neavtive terminal are seperated.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'they created a circut', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the posotive battery and terminal 1 created a circut', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap in the circut', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because of the gap that bulb B created', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive and the negative sides of the battery are separated by the gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because positive charge is before gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'positive before gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'positive connects to negative', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal negative', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is not connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I don\"t know, actually.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because the gap does not affect the posistive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is no gap between the terminals', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the gap', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The posititve terminal is not damaged.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they are flipped', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was connected to the positive side.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'They are different electrical states', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The electrical state at the terminal matches the battery\"s electrical state', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because the voltage meter was connected to a positive and negative side', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was not connect to the negative end of the terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because voltage measures the difference between two points', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I do not know.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'not sure', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'it was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The difference between the electrical states is 1.5 volts', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there was a closed circuit', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the positive terminal is in a closed path.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'it is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal one is connected to a positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was attached to the postive terminal!', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Different states', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a difference of 1.5 volts between terminal 1 and the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 and the positive terminal are connected.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 is connected to the negative terminal which has a 1.5 volt difference with the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the meter was hooked up to two sides of the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 is connected to the negative battery terminal and is in a different state than the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 had both a positive and negative charge.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I\"m not sure', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the positive circuit has a voltage of 1.5 volts.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Postive terminal is not connected to 1.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 was hooked u to the positive terminal which is automatically 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is not connected to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they are connected to opposite terminals', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'beacuse terminal 1 is connected to the negative terminal and the battery is connected at the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is a gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i do not know', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it is not connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is no gap between terminal 1 and the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 is connected to the positive terminal with no gaps', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'It was a mistake. I looked at it wrong.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'We switched the terminals around.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There was a gap in b. The measurement between the positive battery terminal and terminal 1 was 1.5v.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The path is not closed. The distance between the positive battery terminal and terminal a is 1.5v.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive terminal of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there was no gap from the positive terminal of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because 1.5 v minus 0 v equals 1.5 v', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage is the difference between the electrical state of the terminals.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative terminal of the battery and has 0 v, and the positive terminal of the battery has 1.5 v.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the Meter read the difference in volts between the positive battery terminal and terminal 1 which has a negative charge', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Meter read the difference in volts between the positive battery terminal and terminal 1 which is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The gap causes terminal 1 to have a negative battery characteristic which is different compared  to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Bulb terminals 3 and 4', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i don\"t know.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between the circuit.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between terminal 1 and the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap between terminal 1 and the negative battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because there was no gap.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because it was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 had no gap.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 is seperated from the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they were connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is a gap between terminal 3 and 4', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there wasnt a gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage doesnt change', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the gap is open', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the paths are seperated', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the terminal was connected to the positive battery', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the terminals were connected to the positive terminals of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap between the terminals', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 is connected to the negative terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dont know', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was a gap in bulb B', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'circuit was complete until gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'gap, battery and electricity', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'they are in a closed circuit', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i don\"t know', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a gap in the path', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap in the path', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a gap in the circuit', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'don\"t know', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gapp between the positive terminal and terminal 1.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is no gap between terminal 1 and the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'It was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 was connected to the positive terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 was not connected to the negative terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'fjdajf', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal one is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 was connected to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'whatever', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The positive terminal has 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap between the postive terminal and terminal 1', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative end on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative terminal on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal is connected to the bulb', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because 1.5 v is the difference between the positive terminal and terminal 1', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal is connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 is connected to the negative terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The positive terminal and terminal 1 read 1.5 V because the difference across the terminals is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the voltage between terminal 1 and the positive terminal is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative terminal so the voltage reading is 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': '?', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I  do not know', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap after bulb B.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'There is a gap after bulb b', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'All components are in a closed circuit.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the 1.5 volt battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dont know.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 and the neavtive terminal are seperated.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'they created a circut', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the posotive battery and terminal 1 created a circut', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there was no gap in the circut', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because of the gap that bulb B created', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive and the negative sides of the battery are separated by the gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because positive charge is before gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'positive before gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'positive connects to negative', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal negative', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is not connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I don\"t know, actually.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because the gap does not affect the posistive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there is no gap between the terminals', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the gap', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The posititve terminal is not damaged.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because they are flipped', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was connected to the positive side.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'They are different electrical states', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The electrical state at the terminal matches the battery\"s electrical state', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because the voltage meter was connected to a positive and negative side', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was not connect to the negative end of the terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because voltage measures the difference between two points', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I do not know.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'not sure', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'it was connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The difference between the electrical states is 1.5 volts', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because there was a closed circuit', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the positive terminal is in a closed path.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'it is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal one is connected to a positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because it was attached to the postive terminal!', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the negative battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Different states', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'there is a difference of 1.5 volts between terminal 1 and the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 and the positive terminal are connected.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 is connected to the negative terminal which has a 1.5 volt difference with the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the meter was hooked up to two sides of the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Terminal 1 is connected to the negative battery terminal and is in a different state than the positive battery terminal.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because terminal 1 had both a positive and negative charge.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I\"m not sure', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the positive circuit has a voltage of 1.5 volts.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Postive terminal is not connected to 1.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because terminal 1 was hooked u to the positive terminal which is automatically 1.5 V', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is not connected to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap at bulb b, thus affecting the rest of the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb b is damaged there will be a gap in the path at that bulb', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb b is off there will be a gap in the path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The bulb does not effect the others', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B does not effect the other bulbs', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb b is damaged, then the circuit will be incomplete.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is a gap in the connection', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the damaged bulb creates a gap in the circuit.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap in the path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path through the connection of bulb b.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path through bulb b.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb B is damaged a gap is created.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'actually bulbs a and c will be off because the path will no longer be cloed', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap in the path.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path for the other bulbs', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'A and B are off', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap between bulbs', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap between the bulbs', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap in the circuit.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'gap in bulb b', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B has a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path between the bulb-holders', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path between the bulbs', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'damaged bulb b creates a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There is a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the path is not closed anymore', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb B is damaged, the path will have a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb B is damaged, there is no path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There is no path through bulb B', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is a gap in the closed path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is a gap in the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb B is damaged there will be a gap in the curcuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': '?', 'reference_answer': 'bulb B creates a gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They will not be on because it will cause a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb b is damaged, it creates a gap in the circuit.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The path will not be closed.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb b is damaged it does not make a complete circut', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'when bulb b is damaged there is no complete circut', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'distrupt the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because the path connecting the bulbs will be damaged', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the holder of the bulb will damage the other bulb', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because if bulb b is damaged, then there is no path through the bulb, creating an incomplete circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb b is damaged, then the path will have a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'It still has a connect path between bulb a and c and the battery.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are connected to the battery not each other', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'when the bulb is burnt it creates a gap in the closed path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there will be a gap at bulb b. this will create an open circuit.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because they are breaking the circuit.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will not be a closed path.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The path between the bulbs and the battery is open', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because there is no connection so the path is not closed', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'A damaged bulb does not create a gap in a closed path', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb B is damaged there is a gap in the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If buld B is damaged that will create a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Maybe because there is still one path that connects A and C through B', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There will be a gap where the bulb holder is supposed to be', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Burned out bulb B will cause a gap in the closed path.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because there is a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because there is a gap in the bulb', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'it will cut off the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'I\"m not sure.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb B is damaged, then the circut is no longer closed.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb B becomes a gap in the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The bulbs aren\"t codependent so they will still be on', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there would be a gap in the circuit', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the gap between the damaged wires of bulb b creates a gap in the path.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the current will flow to the other bulbs even though it is not able to execute light on B. Also, I\"ve experienced that one light can be out without them all being affected.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because their is a gap between A and C now.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'When bulb B is damaged there is a disruption in the current.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if it is not connected the electrical current can\"t pass through to connect the other bulbs.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There are two pathways', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The circuit will become incomplete.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If bulb B is damaged, there will be a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'There would be a gap.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'it would create a gap', 'reference_answer': 'bulb B creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained within the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb c and a are in the same path. and affect one another', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb c and a are contained in the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'damaging one turns off the other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb a is damaged, bulb c turns off.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'contained within the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a and c affect each other.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a affects bulb c. bulb c affects bulb a.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are both on the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a b and c are on a closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing bulb a will affect bulb c', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a affects bulb c', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on a path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they all affect each other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a, b, and c are on a path with the battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if bulb a burns out if affects bulbs b and c.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a, b, and c are all in the same path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing one bulb shuts off the other.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the path is closed', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because they are in the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'that is all', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs A and C are in the same closed path with the battery.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the are on the same wire', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are not parralel', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they have the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in the same closed path but not next to each other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are in the same path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'That is basically what I said sir.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If one bulb is burns out, then they all stop', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If one bulb is removed, the others stop working', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing one bulb shuts off the other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'i did', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because they are on the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained in the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a and c are wired within a closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs a and c are wired within the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'both bulbs a and c are contained in a closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are contained in the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'kkkdfdadkjlf', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'and the circut is closed', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in a line on the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs A,B, and C are all on the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'tell answer', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained in the same closed path, so if one bulb goes out so do the others.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'The lightbulbs are all contained in the same closed path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are all contained in the same closed path with a battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the bulbs are on the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the bulbs are on the same path with the battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the closed path contains bulb A, bulb B, bulb C, and a battery.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained in the same path. If one bulb goes out it will affect the other ones.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs A and C are contained in the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing of damaging any of the bulbs will shut off the other bulbs.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'i give up', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are all connected in the same path to the battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are in a series of connections with the battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'a and c are on the same closed path with the battery', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing a bulb will turn off another bulb', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs are on the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are located on the same path and will affect each other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs A and C are located on the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are on the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are teh begning and end of the path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they can affect each other', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Because they are connected along the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Each bulb affects the other bulbs', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are on the same path to the battery without interruption', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'becuase there is only one path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'If any of the 3 bulbs are damaged the other 2 bulbs would turn off because they are all within the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A and bulb C are in the same path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same apth', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'wired in same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are all affected by one another along the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'theya re connectted by another bulb', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'THE ARE ON THE SAME PATH', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are all within the same path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are on the same path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they affect one another', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a and c are in a closed circuit', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A and bulb C affect each other.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'becuase breaking one  bulb then affects the ability of the others to light up.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'if one is out the others will go out, they are in the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are all on the dame closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are contained on the same closed path.', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are not parallel', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'removing either A or C will turn off the other bulb', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb a and c were still contained within a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there is still a closed path that contains the battery, bulb a, and bulb c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they are within different paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb a had a seperate closed path from b. bulb c had a seperate closed path from bulb b.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb a is in a closed path, and bulb c is in a closed path.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulbs a and c are on a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb b did not create a gaop.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb a and c were still in a closed path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulbs A and C were still on closed paths with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c are not in the same path of bulb b', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they were contained in closed paths.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because the paths were still in a closed circuit with the battery terminal', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulbs a and c are in separate paths.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because bulb A and C are in a closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'each bulb contains its own closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there are 3 separate closed pat for the 3 bulbs', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulb A and C are still in a closed path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They are in independent paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'still a closed path to the other bulbs', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'the path is closed', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because the bulbs are on seperate paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c are seperate from bulb a', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'exactly', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they each have their own closed paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c have their own closed paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because of the seperate closed paths, one bulb being burned out doesn\"t affect the others.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c are contained in closed paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they are on separate paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they were on closed paths containing the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs A and C are in different paths that do not contain bulb B', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They were still contained in a closed path with the batter', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They were still contained in a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb b did not create a gap for bulb a and c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb b did not create a gap in the path for bulb b or c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb b did not create a gap in the path for bulb a and c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'both bulbs a and c still have a closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there is a closed path to the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'the bulbs are contained in a closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they were still in a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb a and bulb c are not on the same path as bulb B', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb a and bulb c are not on the same path with bulb b', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because bulbs A and C are on different closed paths with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb a and bulb c were still contained in a closed path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there is still a closed path that contains the battery and bulb a and bulb c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb a and bulb c still had a complete path to the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb b did not create a gap in the path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there is still a complete path to bulb a and bulb c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because the bulbs are on separate paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs A and C are on separate paths from bulb B', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c are connected to the battery on the same path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'the battery is contained on a path with bulb a and b', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'the battery is contained on a closed path with bulb a and b', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'the battery contains bulb a and b on a closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because the bulbs are in separate paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they were still contained in a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They were still on path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they do not run on the same current', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they are on their own closed paths with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulbs A and C were still contained on closed paths with the batery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they are on a pathway that is a closed circuit', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They had their own closed paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they were still contained in a closed path to battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they all have different paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because bulbs A and C were in a separate closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulb a and bulb c were still in a closed path with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulb A and C still had a closed path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they still have connection to the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'There is still a closed path for bulb A and C', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They were still in a closed pathway with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'The path was still closed.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they are on seperate circuits', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Different path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'SEPARATE PATHS', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They were not in the same path as bulb B.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulbs A and C were still on their own completed circuts.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they were still contained in closed circuits with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'they were on another closed path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because each bulb has its own path', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'bulbs a and c are still contained in a complete path with the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they were still in a closed circuit with the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They are on separate paths.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Bulb A and bulb C are on closed paths containing the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Becuase they had their own pathways to work on.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they both still had closed paths', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'They all have their own paths connected to the battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they are not on the same closed path.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they are not in the same pathway as bulb b.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because bulbs A and C are still contained in a closed path to the battery.', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'because they are on separate paths than B is', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulb a and c will still be contained within a closed path of the battery', 'reference_answer': 'Bulbs A and C are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is still a closed path that contains the battery, bulb a, and bulb c', 'reference_answer': 'Bulbs A and C are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " ...]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dspy_dataset_best_quality_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "0df7f059-108b-4378-98e9-a5f8bea22aca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we have 5129 data points in total\n"
     ]
    }
   ],
   "source": [
    "print(f\"we have {len(dspy_dataset_best_quality_all)} data points in total\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "b902e8b7-400c-46af-9164-f143071dfd6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a smaller dataset, with 60 examples for each label\n",
    "import random\n",
    "\n",
    "num_examples_per_label = 60\n",
    "\n",
    "# Initialize lists to store examples for each label\n",
    "correct_examples = []\n",
    "partially_correct_examples = []\n",
    "contradictory_examples = []\n",
    "non_domain_examples = []\n",
    "irrelevant_examples = []\n",
    "\n",
    "# Iterate through the original dataset and filter examples by label\n",
    "for example in best_quality_dataset:\n",
    "    label = example[\"label_5way\"]\n",
    "    if label == \"correct\":\n",
    "        correct_examples.append(example)\n",
    "    elif label == \"partially_correct_incomplete\":\n",
    "        partially_correct_examples.append(example)\n",
    "    elif label == \"contradictory\":\n",
    "        contradictory_examples.append(example)\n",
    "    elif label == \"non_domain\":\n",
    "        non_domain_examples.append(example)\n",
    "    elif label == \"irrelevant\":\n",
    "        irrelevant_examples.append(example)\n",
    "\n",
    "# Randomly sample 60 examples for each label\n",
    "sampled_examples = (\n",
    "    random.sample(correct_examples, num_examples_per_label) +\n",
    "    random.sample(partially_correct_examples, num_examples_per_label) +\n",
    "    random.sample(contradictory_examples, num_examples_per_label) +\n",
    "    random.sample(non_domain_examples, num_examples_per_label) +\n",
    "    random.sample(irrelevant_examples, num_examples_per_label)\n",
    ")\n",
    "\n",
    "# Shuffle the sampled examples\n",
    "random.shuffle(sampled_examples)\n",
    "\n",
    "# Create dspy.Example objects for the sampled examples\n",
    "dspy_dataset_equal_distribution = [\n",
    "    dspy.Example(\n",
    "        question=example[\"question\"],\n",
    "        student_answer=example[\"student_answer\"],\n",
    "        reference_answer=example[\"reference_answer\"],\n",
    "        assessment=example[\"label_5way\"]\n",
    "    ).with_inputs(\"question\", \"student_answer\", \"reference_answer\")\n",
    "    for example in sampled_examples\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7571e92f-65bf-4ff1-bba5-f6cbefb0d774",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we have 300 data points in dspy_dataset_equal_distribution\n"
     ]
    }
   ],
   "source": [
    "print(f\"we have {len(dspy_dataset_equal_distribution)} data points in dspy_dataset_equal_distribution\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "95555ca4-93c8-4d3d-8355-429b59addcb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_data, dev_data = train_test_split(dspy_dataset_equal_distribution, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "25d38e26-8d0d-4f53-8429-c97ef2ec14ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal is connected to the bulb', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they have the same path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What role does the path play in determining whether or not a switch affects a bulb?', 'student_answer': 'if the bulb is contained within the path', 'reference_answer': 'If a bulb and a switch are in the same path the switch affects the bulb', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'The circuit is complete with the battery and bulb a', 'reference_answer': 'Bulb A was still contained in the same closed path with the battery.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'Because bulb B is damaged and does not allow the positive battery current to reach the other side.', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'terminal is connected to terminal 6', 'reference_answer': 'Terminals 1 and 6 are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'switch Y does not affect the closed circuit of bulb A', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'your an idiot', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are no longer on a closed cirucit', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'the terminal it is connected to is not working', 'reference_answer': 'the terminals are connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 4 is not a short circuit.', 'student_answer': 'the bulb is not contained in a closed path', 'reference_answer': 'Circuit 4 has no closed paths', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The voltage does not change.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'i dont know', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a damaged/ burned out bulb affect another bulb?', 'student_answer': 'when they are both on the same closed path and would cause a gap', 'reference_answer': 'If they are in the same path the burned out bulb affects the other bulb.', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the terminal will be cloesd', 'reference_answer': 'bulbs B and C are still in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 6 and the positive terminal.', 'student_answer': 'you guys would get more honest feedback and answers if you didnt make this so boring and repeatdly ask the same questions. its not like your testing elementary school kids', 'reference_answer': 'Terminal 6 and the positive terminal are connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Describe the paths in this diagram and explain how those paths account for the results.', 'student_answer': 'im not sure how to', 'reference_answer': \"Bulb A is in a path which does not contain B and C, so bulbs B and C don't affect it. Bulbs B and C are in the same path. They affect each other, but Bulb A doesn't affect them.\", 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'there is no gap between the battery terminal and the bulb terminal.', 'reference_answer': 'the bulb is damaged', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a damaged/ burned out bulb affect another bulb?', 'student_answer': 'if there is a gap in the path', 'reference_answer': 'If they are in the same path the burned out bulb affects the other bulb.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.', 'student_answer': 'The positive battery terminal is seperated by a gap from termnial one.', 'reference_answer': 'Terminal 2 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'i give up', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dint know.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.', 'student_answer': 'The positive battery terminal is separated by a gap from terminal 1. and germinal 3 is connected to the negative battery termina', 'reference_answer': 'Terminal 3 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is no path through the connection of bulb b.', 'reference_answer': 'bulb B creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'Because switch Y and Z are also connected to bulb A.', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 4 is not a short circuit.', 'student_answer': 'i have no idea', 'reference_answer': 'Circuit 4 has no closed paths', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'there will be a gap between the positive battery terminal and the bulb terminal', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a gap.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they are on the same closed path', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'they are seperated by a gap from bulb C', 'reference_answer': 'There is a gap between terminals 1 and 6', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do both bulbs A and B stay on when bulb C is burned out?', 'student_answer': 'bulb c does not create a gap', 'reference_answer': 'If C burns out then A and B are still in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'that there is no electrical connection', 'reference_answer': 'the terminals are connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a switch affect a bulb?', 'student_answer': 'tell answer', 'reference_answer': 'When the switch and the bulb are contained in the same path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The difference between the electrical states is 1.5 volts', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'The battery is in the closed path', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'Where there is a difference in electrical state, that is where the gap in the connection is.', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a damaged bulb.', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'terminal 1 is conected to terminal 6', 'reference_answer': 'There is a gap between terminals 1 and 6', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a switch affect a bulb?', 'student_answer': 'it will turn it off', 'reference_answer': 'When the switch and the bulb are contained in the same path', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'i dont know!!!!', 'reference_answer': 'the terminals are connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'Switch Z will still create a closed path from the bulb and the battery', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'Bulb A is on a different path than switch Y', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'which i mentioned', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a damaged bulb.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'i forgot', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery always has a chemical reaction, positive or negative', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 5 is a short circuit.', 'student_answer': 'battery is connected to itself with no bulb', 'reference_answer': 'the battery is contained in a closed path in which there is no bulb', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because the chemical reaction is not effected by a gap.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.', 'student_answer': 'there is a gap between terminal 3 and the positive terminal', 'reference_answer': 'Terminal 3 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'i dont know', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a gap.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'they are have 0 volts.', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'because it was contained in same path as the battery', 'reference_answer': 'Bulb A was still contained in the same closed path with the battery.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do both bulbs A and B stay on when bulb C is burned out?', 'student_answer': 'how so?', 'reference_answer': 'If C burns out then A and B are still in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because a and b are in a closed circuit still', 'reference_answer': 'bulbs A and B are still in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'Bulb A is in a closed path with the battery.', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'because bulb a can still be in a closed path with the battery if switch z is closed and switch x is closed', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 6 and the positive terminal.', 'student_answer': 'idk', 'reference_answer': 'Terminal 6 and the positive terminal are connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'The gap causes terminal 1 to have a negative battery characteristic which is different compared  to the positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'measure of the difference between the elecrical states of two terminals.', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': '?', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I  do not know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the terminal\"s electrical state is not effected by the incomplete circuit.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'jdkjgkfjg', 'reference_answer': 'the bulb is damaged', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': '...', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'don\"t know', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'a closed path', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'Because it determines the different electrical states between the terminals', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a gap.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'the bulb will not light', 'reference_answer': 'the terminals are connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'terminal is connected to postive end of battery', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What role does the path play in determining whether or not a switch affects a bulb?', 'student_answer': 'bulb is contained in closed path', 'reference_answer': 'If a bulb and a switch are in the same path the switch affects the bulb', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'bulb a and c are contained in a closed path with the battery', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'they have different closed paths', 'reference_answer': 'A and C are in different paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'They are connected to the battery not each other', 'reference_answer': 'bulb B creates a gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.', 'student_answer': 'the positive battery terminal is connected to terminal 4.', 'reference_answer': 'Terminal 5 and the positive terminal are connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 1.5 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'there is a difference in electrical states', 'reference_answer': 'the terminals are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'the battery was contained in a circut', 'reference_answer': 'Bulb A is still contained in a closed path with the battery.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'The path is not closed.', 'reference_answer': 'the open switch creates a gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What role does the path play in determining whether or not a switch affects a bulb?', 'student_answer': 'If the path is not closed, the switch will not affect the bbulb', 'reference_answer': 'If a bulb and a switch are in the same path the switch affects the bulb', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'Closed path between batter and bulb A', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch Z.', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 2 is not a short circuit.', 'student_answer': 'the path of battery 2 is not to itself', 'reference_answer': 'The battery in 2 is not in a closed path', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative terminal on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'It has its own closed path', 'reference_answer': 'Bulb A was still contained in the same closed path with the battery.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 6 and the positive terminal.', 'student_answer': 'i dont know', 'reference_answer': 'Terminal 6 and the positive terminal are connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'because they have the same voltage', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Describe the paths in this diagram and explain how those paths account for the results.', 'student_answer': 'if two bulbs are contained in the same path the damaged bulb affects the other bulb', 'reference_answer': \"Bulb A is in a path which does not contain B and C, so bulbs B and C don't affect it. Bulbs B and C are in the same path. They affect each other, but Bulb A doesn't affect them.\", 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'The bulbs were connected and now they are not.', 'reference_answer': 'the terminals are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do both bulbs A and B stay on when bulb C is burned out?', 'student_answer': 'the bulb in burned out', 'reference_answer': 'If C burns out then A and B are still in a closed path with the battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'if switches X and Z are closed it will complete the closed circuit for bulb A', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'because they have 0 voltz', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'voltage is the difference between states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was open and switch Y was closed, why were bulbs B and C on?', 'student_answer': 'switch x is not in the same path of bulbs b and c.', 'reference_answer': 'Bulb B and Bulb C were still contained in the same closed path with the battery.', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Because the meter was hooked up to two sides of the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'Because it has one regardless of the circuit', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'battery and closed circuit', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'because those terminals are connected to the positive terminal of the battery.', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 5 is a short circuit.', 'student_answer': 'path 5 is a short circuit.', 'reference_answer': 'the battery is contained in a closed path in which there is no bulb', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'l', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.', 'student_answer': 'The positive battery terminal is separated by a gap from terminal 1.', 'reference_answer': 'Terminal 2 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'The terminals are disconnected.', 'reference_answer': 'the terminals are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb C off when switch Z was open?', 'student_answer': 'it was in an open circuit', 'reference_answer': 'There was no longer a closed path containing Bulb C and the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'Because the voltage is the same.', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'a and c are still contained in a closed path with the battery.', 'reference_answer': 'Bulbs A and C are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Voltage is the difference between a positive and negative end on the battery.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'The gap causes a difference in electrical states.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.', 'student_answer': 'Terminal 4 is connected to the positive battery terminal.', 'reference_answer': 'Terminal 5 and the positive terminal are connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'a bulb causes a difference in electrical state', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a gap.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'positive and negative charge', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the are on the same wire', 'reference_answer': 'A and C are in the same closed path', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'Yes it was on', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Describe the paths in this diagram and explain how those paths account for the results.', 'student_answer': 'Bulb B and Bulb C are contained in a closed path.', 'reference_answer': \"Bulb A is in a path which does not contain B and C, so bulbs B and C don't affect it. Bulbs B and C are in the same path. They affect each other, but Bulb A doesn't affect them.\", 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'njo', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'the bulb is damaged', 'reference_answer': 'the bulb is damaged', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 1 and terminal 4.', 'student_answer': 'terminal 1 is connected to terminal 4.', 'reference_answer': 'Terminals 1 and 4 are connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'Both the light bulb and battery must be in a closed path.', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does an open switch impact a circuit?', 'student_answer': 'Because the electricity can\"t pass through', 'reference_answer': 'the open switch creates a gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'an electrical surge', 'reference_answer': 'the terminals are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.', 'student_answer': 'The positive battery terminal is separated by a gap from terminal 1', 'reference_answer': 'Terminal 2 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'don\"t know', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'i dont know', 'reference_answer': 'A and C are in different paths with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'The battery must be connected to the bulb', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'It was in the same path as switch X.', 'reference_answer': 'Bulb A was still contained in the same closed path with the battery.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'bulbs b and c are contiained in the the same path with the battery.', 'reference_answer': 'A and C are in different paths with the battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'a closed component to the battery', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'bulb a will light when it is in a closed path with the battery.', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'you can close switch Z and it will have light', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb A will no longer be completing the circuts.', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 1 and terminal 2.', 'student_answer': 'the terminals are seperated from the positive terminal', 'reference_answer': 'Terminals 1 and 2 are connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because the voltage doesnt change in the battery', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a switch affect a bulb?', 'student_answer': 'when they are contained in the same path of a battery', 'reference_answer': 'When the switch and the bulb are contained in the same path', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a damaged/ burned out bulb affect another bulb?', 'student_answer': 'i don\"t know.', 'reference_answer': 'If they are in the same path the burned out bulb affects the other bulb.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'because of the gap in bulb B', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'bulb a was on', 'reference_answer': 'Bulb A was still contained in the same closed path with the battery.', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'Because they were still contained in a closed path to battery', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 5 is a short circuit.', 'student_answer': 'The battery is in a closed path that does not contain any other components.', 'reference_answer': 'the battery is contained in a path in which there is no bulb', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'because a battery always has a charge in its positive terminal', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'A battery will always have a voltage because the electrical states of the positive and negative terminals are always different.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'positive battery terminal', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do both bulbs A and B stay on when bulb C is burned out?', 'student_answer': 'l', 'reference_answer': 'If C burns out then A and B are still in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What role does the path play in determining whether or not a switch affects a bulb?', 'student_answer': 'The path has to be closed in order for the bulb to be on.', 'reference_answer': 'If a bulb and a switch are in the same path the switch affects the bulb', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'wow', 'reference_answer': 'the bulb is damaged', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 is connected to the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'Because the switch was connected to the closed circuit', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulbs A and B will still be contained in a closed path to the battery.', 'reference_answer': 'bulbs A and B are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'The difference in electrical charge between two terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'there is still a closed path that contains the battery, bulb a, and bulb b', 'reference_answer': 'bulbs A and B are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb C off when switch Z was open?', 'student_answer': 'Bulb B is in the path of the opened switch', 'reference_answer': 'There was no longer a closed path containing Bulb C and the battery', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'since a a gap does not affect the voltage of a battery', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'A terminal is connected to a bulb.', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'A is on a battery that has a closed path', 'reference_answer': 'Bulb A is still contained in a closed path with the battery.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'there is a gap between the positive battery terminal and the bulb terminal.', 'reference_answer': 'the terminals are connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'I accounted for B when it wasn\"t pertinant to the question', 'reference_answer': 'A and C are in different paths with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'current is present', 'reference_answer': 'the terminals are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'is the difference between terminals', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 4 is not a short circuit.', 'student_answer': 'Because neither of the components are connected to themselves.', 'reference_answer': 'Circuit 4 has no closed paths', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'I have no idea', 'reference_answer': 'Bulb A is still contained in a closed path with the battery.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 1 and terminal 2.', 'student_answer': 'they are connected', 'reference_answer': 'Terminals 1 and 2 are connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do both bulbs A and B stay on when bulb C is burned out?', 'student_answer': 'bulbs a and b are still in a closed path with the battery', 'reference_answer': 'If C burns out then A and B are still in a closed path with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.', 'student_answer': 'idk', 'reference_answer': 'Terminal 2 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i do not understand', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why circuit 4 is not a short circuit.', 'student_answer': 'the bulb is not connected to the positive connection.', 'reference_answer': 'Circuit 4 has no closed paths', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': '?', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a damaged bulb.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because 1.5 v minus 0 v equals 1.5 v', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'tell answer', 'reference_answer': 'If bulb A burns out, B and C are no longer in a closed path with the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.', 'student_answer': 'l', 'reference_answer': 'Measuring voltage indicates the place where the electrical state changes due to a gap.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What do you think the red X means?', 'student_answer': 'don\"t do it.', 'reference_answer': 'the battery is damaged', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a switch affect a bulb?', 'student_answer': 'l', 'reference_answer': 'When the switch and the bulb are contained in the same path', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'Because terminal 1 and 6 are connceted to the battery.', 'reference_answer': 'Terminals 1 and 6 are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What role does the path play in determining whether or not a switch affects a bulb?', 'student_answer': 'tell answer', 'reference_answer': 'If a bulb and a switch are in the same path the switch affects the bulb', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What does a voltage reading of 0 tell you about the connection between a bulb terminal and a battery terminal?', 'student_answer': 'a voltage reading of 0 volts indicates a gap in the circuit.', 'reference_answer': 'the terminals are connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why didn^t bulbs A and C go out after bulb B burned out?', 'student_answer': 'there is still a complete path to bulb a and bulb c', 'reference_answer': 'Bulbs A and C are still contained in closed paths with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb C off when switch Z was open?', 'student_answer': 'the aijfkfthgh', 'reference_answer': 'There was no longer a closed path containing Bulb C and the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'difference in electrial state', 'reference_answer': 'There is a gap between terminals 1 and 6', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 4 and the positive terminal.', 'student_answer': 'positive battery terminal is not separated by a gap from terminal 4', 'reference_answer': 'Terminal 4 and the positive terminal are connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.', 'student_answer': 'terminal 1 is not connected to terminal 6', 'reference_answer': 'Terminals 1 and 6 are separated by the gap', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'because voltage measures the difference between two points', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 1 and terminal 4.', 'student_answer': 'They are both connected to the negative battery terminal', 'reference_answer': 'Terminals 1 and 4 are connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a damaged/ burned out bulb affect another bulb?', 'student_answer': 'if bulb A burns out', 'reference_answer': 'If they are in the same path the burned out bulb affects the other bulb.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the terminals were connected to the positive terminals of the battery', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.', 'student_answer': 'no gap', 'reference_answer': 'Terminal 5 and the positive terminal are connected', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'i dont know', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What are the conditions that are required to make a bulb light up?', 'student_answer': 'i dont know', 'reference_answer': 'there is a closed path containing both the bulb and a battery', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.', 'student_answer': 'positive battery terminal is separated by a gap from terminal 3', 'reference_answer': 'Terminal 3 and the positive terminal are not connected', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'battery always has voltage', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 and the positive terminal are connected.', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'its connected', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'there terminals are separated by a gap when the voltage jumps from 0 to 1.5.', 'reference_answer': 'the bulb is damaged', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the measurement of the difference between two terminals in an electric state', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'There is a difference in electrical states.', 'reference_answer': 'the bulb is damaged', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'Different electrical states.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'When switch X was closed and switch Y was open, why was bulb A on?', 'student_answer': 'l', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch X.', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'the bulbs are on separate paths', 'reference_answer': 'bulbs A and B are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?', 'student_answer': 'i dont know', 'reference_answer': 'the terminals are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.', 'student_answer': 'same answer as above', 'reference_answer': 'Terminal 3 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'because bulbs a and b have their own closed paths', 'reference_answer': 'bulbs A and B are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'the amount of volts', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Under what circumstances will a damaged/ burned out bulb affect another bulb?', 'student_answer': 'Bulbs have to be contained in closed paths with the battery', 'reference_answer': 'If they are in the same path the burned out bulb affects the other bulb.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'the positive terminal', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'irrelevant'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'because the negative current can transfer through switch z', 'reference_answer': 'Bulb A is still in a closed path with the battery', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why?', 'student_answer': 'the voltage of the battery does not depend on the components in a curcuit. It depends on a chemical reaction.', 'reference_answer': 'A battery uses a chemical reaction to maintain different electrical states at the terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'It had its own path', 'reference_answer': 'Bulb A is still contained in a closed path with the battery.', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'What is voltage?', 'student_answer': 'Two terminals that have different electrical states', 'reference_answer': 'Voltage is the difference in electrical states between two terminals', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'terminal 1 and the positive terminal are connected.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'I don\"t know, actually.', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why was bulb A on when switch Y was open and switch Z was closed?', 'student_answer': 'A is connected to the positive terminal.', 'reference_answer': 'Bulb A is still contained in a closed path with the battery and switch Z.', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'Bulb a is in a separate path', 'reference_answer': 'bulbs B and C are still in closed paths with the battery', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think the other terminals are being held in a different electrical state than that of the negative terminal?', 'student_answer': 'Because they are connected to the positive terminal.', 'reference_answer': 'Terminals 4, 5 and 6 are not connected to the negative battery terminal', 'assessment': 'correct'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why not?', 'student_answer': 'the bulb could be lit by having switch X and switch Z closed.', 'reference_answer': 'there is still a closed path containing Switch Z, Switch X, Bulbs A and C and the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain your reasoning.', 'student_answer': 'all the bulbs are on parralel paths', 'reference_answer': 'A and C are in different paths with the battery', 'assessment': 'partially_correct_incomplete'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why does a damaged bulb impact a circuit?', 'student_answer': 'it blocks the flow of electricity', 'reference_answer': 'a damaged bulb creates a gap', 'assessment': 'contradictory'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'i dont know', 'reference_answer': 'Terminal 1 and the positive terminal are separated by the gap', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Why do you think those terminals and the negative battery terminal are in the same state?', 'student_answer': 'I don\"t know', 'reference_answer': 'Terminals 1, 2 and 3 are connected to the negative battery terminal', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'}),\n",
       " Example({'question': 'Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.', 'student_answer': 'not sure', 'reference_answer': 'Terminal 1 and the positive terminal are not connected', 'assessment': 'non_domain'}) (input_keys={'student_answer', 'reference_answer', 'question'})]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9494af3-a74e-48a3-80b2-94be80b6cb56",
   "metadata": {},
   "source": [
    "## Building a metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "52efdd2a-fa00-4520-aeee-15f49ddef525",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dspy\n",
    "\n",
    "def grading_accuracy_metric(example, predicted, trace=None):\n",
    "    \n",
    "    score = 1 if example[\"assessment\"] == predicted[\"assessment\"] else 0\n",
    "    \n",
    "    return score\n",
    "\n",
    "from dspy.evaluate import Evaluate\n",
    "\n",
    "EM_metric = dspy.evaluate.answer_exact_match"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fbadbc7-1310-4f87-a25d-49b3c5aa0aa8",
   "metadata": {},
   "source": [
    "## Optimising Few Shot Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4dd7c9e0-394b-4e8c-8c84-f10069ad9ba4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 48 / 150  (32.0):  71%|▋| 150/210 [01:58<01:01,  1.02s/itERROR:dspy.primitives.assertions:\u001b[2m2024-05-17T15:10:04.854661Z\u001b[0m [\u001b[31m\u001b[1merror    \u001b[0m] \u001b[1mSuggestionFailed: 'partially_correct_incomplete_contradictory' must be exactly one of 'correct', 'partially_correct_incomplete', 'contradictory', 'non_domain', 'irrelevant'\u001b[0m [\u001b[0m\u001b[1m\u001b[34mdspy.primitives.assertions\u001b[0m]\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35massertions.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m111\u001b[0m\n",
      "ERROR:dspy.evaluate.evaluate:\u001b[2m2024-05-17T15:10:04.857115Z\u001b[0m [\u001b[31m\u001b[1merror    \u001b[0m] \u001b[1mError for example in dev set: \t\t 'partially_correct_incomplete_contradictory' must be exactly one of 'correct', 'partially_correct_incomplete', 'contradictory', 'non_domain', 'irrelevant'\u001b[0m [\u001b[0m\u001b[1m\u001b[34mdspy.evaluate.evaluate\u001b[0m]\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mevaluate.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m147\u001b[0m\n",
      "Average Metric: 77.0 / 210  (36.7): 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [02:43<00:00,  1.28it/s]\n",
      "Average Metric: 36 / 88  (40.9):  42%|█████████████████████████████████████████████████████████████████████████████▌                                                                                                           | 88/210 [01:05<01:50,  1.10it/s]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88384, Requested 3364. Please try again in 1.165s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 37 / 90  (41.1):  43%|███████████████████████████████████████████████████████████████████████████████▎                                                                                                         | 90/210 [01:08<02:18,  1.15s/it]INFO:backoff:Backing off request(...) for 1.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88349, Requested 3364. Please try again in 1.142s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.3 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 37 / 96  (38.5):  46%|████████████████████████████████████████████████████████████████████████████████████▌                                                                                                    | 96/210 [01:16<02:19,  1.23s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88348, Requested 3370. Please try again in 1.145s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 38 / 98  (38.8):  47%|██████████████████████████████████████████████████████████████████████████████████████▎                                                                                                  | 98/210 [01:19<02:35,  1.39s/it]INFO:backoff:Backing off request(...) for 0.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88365, Requested 3370. Please try again in 1.156s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.1 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 39 / 108  (36.1):  51%|██████████████████████████████████████████████████████████████████████████████████████████████                                                                                         | 108/210 [01:33<02:06,  1.24s/it]INFO:backoff:Backing off request(...) for 0.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88320, Requested 3397. Please try again in 1.144s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 43 / 115  (37.4):  55%|████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                  | 115/210 [01:42<01:48,  1.15s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88318, Requested 3370. Please try again in 1.125s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 46 / 118  (39.0):  56%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                | 118/210 [01:46<01:51,  1.21s/it]INFO:backoff:Backing off request(...) for 1.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88365, Requested 3370. Please try again in 1.156s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 46 / 122  (37.7):  58%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                            | 122/210 [01:52<02:09,  1.47s/it]INFO:backoff:Backing off request(...) for 0.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88436, Requested 3348. Please try again in 1.189s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 46 / 123  (37.4):  59%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                           | 123/210 [01:55<02:29,  1.72s/it]INFO:backoff:Backing off request(...) for 2.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88831, Requested 3348. Please try again in 1.452s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.0 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 47 / 127  (37.0):  60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                        | 127/210 [02:01<02:13,  1.60s/it]INFO:backoff:Backing off request(...) for 3.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88395, Requested 3348. Please try again in 1.162s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 48 / 133  (36.1):  63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                                   | 133/210 [02:09<01:40,  1.30s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88352, Requested 3358. Please try again in 1.14s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 48 / 136  (35.3):  65%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                | 136/210 [02:13<01:37,  1.32s/it]INFO:backoff:Backing off request(...) for 1.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88366, Requested 3358. Please try again in 1.149s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.3 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50 / 139  (36.0):  66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                             | 139/210 [02:17<01:31,  1.29s/it]INFO:backoff:Backing off request(...) for 3.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88364, Requested 3358. Please try again in 1.148s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.7 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 51 / 148  (34.5):  70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                      | 148/210 [02:31<01:23,  1.35s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88378, Requested 3371. Please try again in 1.166s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 54 / 152  (35.5):  72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                  | 152/210 [02:37<01:10,  1.21s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88365, Requested 3352. Please try again in 1.144s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 56 / 156  (35.9):  74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                               | 156/210 [02:41<01:01,  1.14s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 87096, Requested 3352. Please try again in 298ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 58 / 158  (36.7):  75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                             | 158/210 [02:45<01:16,  1.47s/it]INFO:backoff:Backing off request(...) for 2.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88380, Requested 3352. Please try again in 1.154s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.9 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 59 / 162  (36.4):  77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                         | 162/210 [02:51<01:11,  1.48s/it]INFO:backoff:Backing off request(...) for 2.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88375, Requested 3352. Please try again in 1.151s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.7 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 61 / 167  (36.5):  80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                     | 167/210 [02:59<01:05,  1.52s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88335, Requested 3368. Please try again in 1.135s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 62 / 168  (36.9):  80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                    | 168/210 [03:00<00:58,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 62 / 170  (36.5):  81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 170/210 [03:03<01:00,  1.50s/it]INFO:backoff:Backing off request(...) for 1.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88378, Requested 3368. Please try again in 1.164s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.0 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 63 / 173  (36.4):  82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                | 173/210 [03:07<00:51,  1.39s/it]INFO:backoff:Backing off request(...) for 3.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88392, Requested 3368. Please try again in 1.173s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.5 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 68 / 178  (38.2):  85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                            | 178/210 [03:14<00:45,  1.42s/it]INFO:backoff:Backing off request(...) for 5.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88368, Requested 3368. Please try again in 1.157s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 5.0 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 70 / 184  (38.0):  88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                      | 184/210 [03:23<00:32,  1.26s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88355, Requested 3375. Please try again in 1.153s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 71 / 188  (37.8):  90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                   | 188/210 [03:28<00:29,  1.35s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88377, Requested 3353. Please try again in 1.153s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 72 / 190  (37.9):  90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                 | 190/210 [03:31<00:23,  1.19s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88382, Requested 3353. Please try again in 1.156s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 72 / 191  (37.7):  91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                | 191/210 [03:32<00:22,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 73 / 193  (37.8):  92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏              | 193/210 [03:35<00:24,  1.43s/it]INFO:backoff:Backing off request(...) for 1.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88381, Requested 3353. Please try again in 1.156s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 74 / 196  (37.8):  93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊            | 196/210 [03:39<00:20,  1.44s/it]INFO:backoff:Backing off request(...) for 1.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88353, Requested 3353. Please try again in 1.137s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.5 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 77 / 199  (38.7):  95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 199/210 [03:43<00:15,  1.38s/it]INFO:backoff:Backing off request(...) for 10.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88366, Requested 3353. Please try again in 1.146s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 10.0 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 83 / 210  (39.5): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [04:00<00:00,  1.15s/it]\n",
      "  9%|███████████████████▋                                                                                                                                                                                                      | 19/210 [00:02<00:25,  7.46it/s]\n",
      "Average Metric: 1 / 3  (33.3):   1%|██▋                                                                                                                                                                                         | 3/210 [00:05<06:08,  1.78s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86143, Requested 4280. Please try again in 282ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 3 / 5  (60.0):   2%|████▍                                                                                                                                                                                       | 5/210 [00:09<06:42,  1.97s/it]INFO:backoff:Backing off request(...) for 1.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88143, Requested 4280. Please try again in 1.615s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 3 / 6  (50.0):   3%|█████▎                                                                                                                                                                                      | 6/210 [00:10<05:57,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.4 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 4 / 8  (50.0):   4%|███████▏                                                                                                                                                                                    | 8/210 [00:14<06:21,  1.89s/it]INFO:backoff:Backing off request(...) for 3.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88141, Requested 4280. Please try again in 1.614s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 4 / 9  (44.4):   4%|████████                                                                                                                                                                                    | 9/210 [00:16<06:01,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.6 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 7 / 13  (53.8):   6%|███████████▌                                                                                                                                                                              | 13/210 [00:23<06:02,  1.84s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88097, Requested 4313. Please try again in 1.606s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 8 / 16  (50.0):   8%|██████████████▏                                                                                                                                                                           | 16/210 [00:29<06:00,  1.86s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88064, Requested 4336. Please try again in 1.6s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 8 / 17  (47.1):   8%|███████████████                                                                                                                                                                           | 17/210 [00:31<05:55,  1.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 9 / 19  (47.4):   9%|████████████████▊                                                                                                                                                                         | 19/210 [00:34<05:39,  1.78s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89594, Requested 4336. Please try again in 2.62s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 11 / 22  (50.0):  10%|███████████████████▍                                                                                                                                                                     | 22/210 [00:40<05:43,  1.83s/it]INFO:backoff:Backing off request(...) for 3.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86572, Requested 4336. Please try again in 605ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.8 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 12 / 26  (46.2):  12%|██████████████████████▉                                                                                                                                                                  | 26/210 [00:47<05:39,  1.84s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86556, Requested 4336. Please try again in 594ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 12 / 28  (42.9):  13%|████████████████████████▋                                                                                                                                                                | 28/210 [00:51<05:39,  1.86s/it]INFO:backoff:Backing off request(...) for 10.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88110, Requested 4336. Please try again in 1.63s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 13 / 29  (44.8):  14%|█████████████████████████▌                                                                                                                                                               | 29/210 [00:53<05:37,  1.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 10.0 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 15 / 36  (41.7):  17%|███████████████████████████████▋                                                                                                                                                         | 36/210 [01:06<05:17,  1.82s/it]INFO:backoff:Backing off request(...) for 13.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88077, Requested 4336. Please try again in 1.608s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 16 / 37  (43.2):  18%|████████████████████████████████▌                                                                                                                                                        | 37/210 [01:08<05:15,  1.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 13.6 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 21 / 46  (45.7):  22%|████████████████████████████████████████▌                                                                                                                                                | 46/210 [01:24<05:04,  1.85s/it]INFO:backoff:Backing off request(...) for 6.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89596, Requested 4336. Please try again in 2.621s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.0 seconds after 7 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 23 / 51  (45.1):  24%|████████████████████████████████████████████▉                                                                                                                                            | 51/210 [01:34<05:12,  1.97s/it]INFO:backoff:Backing off request(...) for 79.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88088, Requested 4336. Please try again in 1.616s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 79.5 seconds after 8 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 96  (42.7):  46%|████████████████████████████████████████████████████████████████████████████████████▌                                                                                                    | 96/210 [02:57<03:39,  1.93s/it]INFO:backoff:Backing off request(...) for 24.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88055, Requested 4336. Please try again in 1.594s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 24.4 seconds after 9 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 45 / 112  (40.2):  53%|█████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                                     | 112/210 [03:26<02:52,  1.76s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89619, Requested 4306. Please try again in 2.616s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 46 / 115  (40.0):  55%|████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                  | 115/210 [03:32<02:48,  1.78s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86521, Requested 4311. Please try again in 554ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 47 / 116  (40.5):  55%|█████████████████████████████████████████████████████████████████████████████████████████████████████                                                                                  | 116/210 [03:34<02:46,  1.77s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88090, Requested 4311. Please try again in 1.6s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 49 / 119  (41.2):  57%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                               | 119/210 [03:39<02:49,  1.86s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88098, Requested 4311. Please try again in 1.606s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 49 / 121  (40.5):  58%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                             | 121/210 [03:44<02:55,  1.98s/it]INFO:backoff:Backing off request(...) for 3.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88070, Requested 4311. Please try again in 1.587s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.6 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50 / 125  (40.0):  60%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                                          | 125/210 [03:51<02:47,  1.97s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88111, Requested 4311. Please try again in 1.614s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 52 / 129  (40.3):  61%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                       | 128/210 [03:58<02:33,  1.87s/it]INFO:backoff:Backing off request(...) for 30.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86557, Requested 4311. Please try again in 578ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 52 / 129  (40.3):  61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                      | 129/210 [03:58<02:23,  1.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 30.0 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 57 / 148  (38.5):  70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                      | 148/210 [04:33<01:49,  1.76s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86555, Requested 4301. Please try again in 570ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 58 / 150  (38.7):  71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                    | 150/210 [04:37<01:48,  1.81s/it]INFO:backoff:Backing off request(...) for 1.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88092, Requested 4301. Please try again in 1.595s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 59 / 151  (39.1):  72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                   | 151/210 [04:39<01:47,  1.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 60 / 153  (39.2):  73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                 | 153/210 [04:42<01:48,  1.91s/it]INFO:backoff:Backing off request(...) for 2.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88148, Requested 4301. Please try again in 1.632s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.9 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 62 / 157  (39.5):  75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                              | 157/210 [04:49<01:34,  1.78s/it]INFO:backoff:Backing off request(...) for 3.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89147, Requested 4301. Please try again in 2.298s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.3 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 65 / 161  (40.4):  77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                          | 161/210 [04:57<01:31,  1.87s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89594, Requested 4301. Please try again in 2.596s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 66 / 163  (40.5):  78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                         | 163/210 [05:00<01:26,  1.83s/it]INFO:backoff:Backing off request(...) for 8.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88099, Requested 4301. Please try again in 1.6s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 66 / 164  (40.2):  78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                        | 164/210 [05:02<01:26,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 8.6 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 69 / 170  (40.6):  81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 170/210 [05:15<01:26,  2.16s/it]INFO:backoff:Backing off request(...) for 55.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88121, Requested 4301. Please try again in 1.614s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 55.8 seconds after 7 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 86 / 202  (42.6):  96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████       | 202/210 [06:13<00:14,  1.81s/it]INFO:backoff:Backing off request(...) for 80.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88138, Requested 4301. Please try again in 1.626s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 80.5 seconds after 8 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 91 / 210  (43.3): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [07:38<00:00,  2.18s/it]\n",
      "  4%|████████▎                                                                                                                                                                                                                  | 8/210 [00:11<04:56,  1.47s/it]\n",
      "Average Metric: 26 / 59  (44.1):  28%|███████████████████████████████████████████████████▉                                                                                                                                     | 59/210 [00:40<02:14,  1.12it/s]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88200, Requested 3907. Please try again in 1.404s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 29 / 62  (46.8):  30%|██████████████████████████████████████████████████████▌                                                                                                                                  | 62/210 [00:45<03:07,  1.26s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88193, Requested 3907. Please try again in 1.4s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 32 / 65  (49.2):  31%|█████████████████████████████████████████████████████████▎                                                                                                                               | 65/210 [00:50<03:44,  1.55s/it]INFO:backoff:Backing off request(...) for 3.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88231, Requested 3907. Please try again in 1.425s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 32 / 69  (46.4):  33%|████████████████████████████████████████████████████████████▊                                                                                                                            | 69/210 [00:57<03:54,  1.66s/it]INFO:backoff:Backing off request(...) for 5.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88202, Requested 3907. Please try again in 1.406s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 5.6 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 35 / 75  (46.7):  36%|██████████████████████████████████████████████████████████████████                                                                                                                       | 75/210 [01:06<03:40,  1.63s/it]INFO:backoff:Backing off request(...) for 1.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88190, Requested 3907. Please try again in 1.398s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.6 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 35 / 78  (44.9):  37%|████████████████████████████████████████████████████████████████████▋                                                                                                                    | 78/210 [01:11<03:37,  1.65s/it]INFO:backoff:Backing off request(...) for 7.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88194, Requested 3907. Please try again in 1.4s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 7.7 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 40 / 87  (46.0):  41%|████████████████████████████████████████████████████████████████████████████▋                                                                                                            | 87/210 [01:26<03:18,  1.61s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86741, Requested 3912. Please try again in 435ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 40 / 90  (44.4):  43%|███████████████████████████████████████████████████████████████████████████████▎                                                                                                         | 90/210 [01:30<03:04,  1.54s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86740, Requested 3912. Please try again in 434ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 40 / 91  (44.0):  43%|████████████████████████████████████████████████████████████████████████████████▏                                                                                                        | 91/210 [01:32<03:10,  1.60s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88249, Requested 3905. Please try again in 1.436s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 40 / 92  (43.5):  44%|█████████████████████████████████████████████████████████████████████████████████                                                                                                        | 92/210 [01:34<03:14,  1.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 94  (43.6):  45%|██████████████████████████████████████████████████████████████████████████████████▊                                                                                                      | 94/210 [01:37<03:02,  1.57s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86742, Requested 3901. Please try again in 428ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 95  (43.2):  45%|███████████████████████████████████████████████████████████████████████████████████▋                                                                                                     | 95/210 [01:39<03:11,  1.66s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88204, Requested 3919. Please try again in 1.415s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 41 / 96  (42.7):  46%|████████████████████████████████████████████████████████████████████████████████████▌                                                                                                    | 96/210 [01:41<03:15,  1.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 98  (41.8):  47%|██████████████████████████████████████████████████████████████████████████████████████▎                                                                                                  | 98/210 [01:44<03:13,  1.73s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88192, Requested 3919. Please try again in 1.407s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 41 / 99  (41.4):  47%|███████████████████████████████████████████████████████████████████████████████████████▏                                                                                                 | 99/210 [01:46<03:05,  1.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 100  (41.0):  48%|███████████████████████████████████████████████████████████████████████████████████████▏                                                                                               | 100/210 [01:47<02:55,  1.60s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88182, Requested 3921. Please try again in 1.402s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 41 / 101  (40.6):  48%|████████████████████████████████████████████████████████████████████████████████████████                                                                                               | 101/210 [01:49<03:04,  1.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 42 / 103  (40.8):  49%|█████████████████████████████████████████████████████████████████████████████████████████▊                                                                                             | 103/210 [01:52<02:52,  1.61s/it]INFO:backoff:Backing off request(...) for 0.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86781, Requested 3915. Please try again in 464ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.1 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 43 / 105  (41.0):  50%|███████████████████████████████████████████████████████████████████████████████████████████▌                                                                                           | 105/210 [01:55<02:41,  1.54s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86779, Requested 3906. Please try again in 456ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 44 / 107  (41.1):  51%|█████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                         | 107/210 [01:59<03:10,  1.85s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88205, Requested 3906. Please try again in 1.407s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 45 / 110  (40.9):  52%|███████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                       | 110/210 [02:04<02:41,  1.62s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88206, Requested 3906. Please try again in 1.408s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 46 / 114  (40.4):  54%|███████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                                   | 114/210 [02:10<02:20,  1.47s/it]INFO:backoff:Backing off request(...) for 2.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86766, Requested 3906. Please try again in 448ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.2 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 49 / 117  (41.9):  56%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                                                 | 117/210 [02:15<02:34,  1.66s/it]INFO:backoff:Backing off request(...) for 5.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88203, Requested 3906. Please try again in 1.406s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 50 / 118  (42.4):  56%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                | 118/210 [02:17<02:26,  1.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 5.3 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50 / 122  (41.0):  58%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                            | 122/210 [02:24<02:39,  1.81s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88195, Requested 3893. Please try again in 1.392s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50 / 124  (40.3):  59%|████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                                           | 124/210 [02:28<02:41,  1.87s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88205, Requested 3893. Please try again in 1.398s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 51 / 127  (40.2):  60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                        | 127/210 [02:33<02:22,  1.72s/it]INFO:backoff:Backing off request(...) for 2.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88228, Requested 3893. Please try again in 1.414s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.8 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 52 / 132  (39.4):  63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                                    | 132/210 [02:40<02:02,  1.57s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88215, Requested 3906. Please try again in 1.414s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 53 / 135  (39.3):  64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                 | 135/210 [02:45<02:04,  1.66s/it]INFO:backoff:Backing off request(...) for 1.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88194, Requested 3906. Please try again in 1.4s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 53 / 136  (39.0):  65%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                | 136/210 [02:47<01:59,  1.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 55 / 138  (39.9):  66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                              | 138/210 [02:50<01:54,  1.60s/it]INFO:backoff:Backing off request(...) for 2.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88111, Requested 3906. Please try again in 1.344s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.9 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 55 / 143  (38.5):  68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                          | 143/210 [02:58<01:44,  1.56s/it]INFO:backoff:Backing off request(...) for 2.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86770, Requested 3906. Please try again in 450ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.4 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 55 / 146  (37.7):  70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                       | 146/210 [03:03<01:45,  1.65s/it]INFO:backoff:Backing off request(...) for 15.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88218, Requested 3906. Please try again in 1.416s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 15.0 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 60 / 157  (38.2):  75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                              | 157/210 [03:21<01:29,  1.69s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88226, Requested 3899. Please try again in 1.416s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 61 / 158  (38.6):  75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                             | 158/210 [03:23<01:25,  1.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 63 / 160  (39.4):  76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                           | 160/210 [03:26<01:19,  1.59s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89610, Requested 3899. Please try again in 2.339s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 64 / 162  (39.5):  77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                         | 162/210 [03:29<01:20,  1.68s/it]INFO:backoff:Backing off request(...) for 2.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88209, Requested 3899. Please try again in 1.405s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 65 / 163  (39.9):  78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                         | 163/210 [03:31<01:16,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.9 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 66 / 166  (39.8):  79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                      | 166/210 [03:36<01:12,  1.65s/it]INFO:backoff:Backing off request(...) for 4.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88191, Requested 3899. Please try again in 1.393s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 4.8 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 69 / 171  (40.4):  81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                  | 171/210 [03:44<01:04,  1.65s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88232, Requested 3907. Please try again in 1.426s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 70 / 174  (40.2):  83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                               | 174/210 [03:49<01:01,  1.72s/it]INFO:backoff:Backing off request(...) for 1.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88228, Requested 3907. Please try again in 1.423s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.5 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 71 / 177  (40.1):  84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                            | 177/210 [03:54<00:53,  1.63s/it]INFO:backoff:Backing off request(...) for 2.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88219, Requested 3907. Please try again in 1.417s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 72 / 178  (40.4):  85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                            | 178/210 [03:56<00:52,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.1 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 73 / 182  (40.1):  87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                        | 182/210 [04:02<00:41,  1.48s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86759, Requested 3897. Please try again in 437ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 74 / 184  (40.2):  88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                      | 184/210 [04:06<00:45,  1.74s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88202, Requested 3897. Please try again in 1.399s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 75 / 185  (40.5):  88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                     | 185/210 [04:07<00:40,  1.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 76 / 188  (40.4):  90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                   | 188/210 [04:12<00:34,  1.58s/it]INFO:backoff:Backing off request(...) for 3.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86759, Requested 3897. Please try again in 437ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 77 / 192  (40.1):  91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 192/210 [04:19<00:29,  1.63s/it]INFO:backoff:Backing off request(...) for 5.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86743, Requested 3897. Please try again in 426ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 5.3 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 80 / 197  (40.6):  94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋           | 197/210 [04:27<00:20,  1.58s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89615, Requested 3918. Please try again in 2.355s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 82 / 199  (41.2):  95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 199/210 [04:30<00:17,  1.59s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88168, Requested 3918. Please try again in 1.39s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 83 / 200  (41.5):  95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 200/210 [04:32<00:16,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 83 / 202  (41.1):  96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████       | 202/210 [04:35<00:12,  1.59s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89625, Requested 3918. Please try again in 2.362s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 83 / 204  (40.7):  97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 204/210 [04:38<00:09,  1.56s/it]INFO:backoff:Backing off request(...) for 6.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89629, Requested 3918. Please try again in 2.364s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.4 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 88 / 210  (41.9): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [04:49<00:00,  1.38s/it]\n",
      "  8%|█████████████████▋                                                                                                                                                                                                        | 17/210 [00:28<05:21,  1.67s/it]\n",
      "Average Metric: 6 / 12  (50.0):   6%|██████████▋                                                                                                                                                                               | 12/210 [00:12<03:24,  1.03s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88057, Requested 3550. Please try again in 1.071s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 7 / 15  (46.7):   7%|█████████████▎                                                                                                                                                                            | 15/210 [00:17<04:26,  1.37s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88305, Requested 3550. Please try again in 1.236s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 9 / 18  (50.0):   9%|███████████████▉                                                                                                                                                                          | 18/210 [00:21<04:48,  1.50s/it]INFO:backoff:Backing off request(...) for 3.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88290, Requested 3550. Please try again in 1.226s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 11 / 22  (50.0):  10%|███████████████████▍                                                                                                                                                                     | 22/210 [00:27<04:32,  1.45s/it]INFO:backoff:Backing off request(...) for 6.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88276, Requested 3550. Please try again in 1.217s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.0 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 13 / 29  (44.8):  14%|█████████████████████████▌                                                                                                                                                               | 29/210 [00:37<04:39,  1.54s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88328, Requested 3550. Please try again in 1.252s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 14 / 31  (45.2):  15%|███████████████████████████▎                                                                                                                                                             | 31/210 [00:40<04:38,  1.56s/it]INFO:backoff:Backing off request(...) for 24.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88291, Requested 3550. Please try again in 1.227s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 24.2 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 23 / 50  (46.0):  24%|████████████████████████████████████████████                                                                                                                                             | 50/210 [01:07<03:39,  1.37s/it]INFO:backoff:Backing off request(...) for 22.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89163, Requested 3550. Please try again in 1.808s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 22.9 seconds after 7 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 31 / 68  (45.6):  32%|███████████████████████████████████████████████████████████▉                                                                                                                             | 68/210 [01:34<03:24,  1.44s/it]INFO:backoff:Backing off request(...) for 127.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88240, Requested 3550. Please try again in 1.193s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 127.7 seconds after 8 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 60 / 159  (37.7):  76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                            | 159/210 [03:48<00:49,  1.03it/s]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88263, Requested 3577. Please try again in 1.226s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 62 / 162  (38.3):  77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                         | 162/210 [03:51<00:50,  1.06s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89611, Requested 3575. Please try again in 2.124s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 63 / 164  (38.4):  78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                        | 164/210 [03:55<01:00,  1.32s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89635, Requested 3575. Please try again in 2.14s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 64 / 166  (38.6):  79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                      | 166/210 [03:58<01:06,  1.52s/it]INFO:backoff:Backing off request(...) for 2.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88252, Requested 3575. Please try again in 1.218s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.0 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 67 / 170  (39.4):  81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 170/210 [04:04<00:56,  1.41s/it]INFO:backoff:Backing off request(...) for 6.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88263, Requested 3575. Please try again in 1.225s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 67 / 171  (39.2):  81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                  | 171/210 [04:05<00:52,  1.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.1 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 70 / 176  (39.8):  84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                             | 176/210 [04:13<00:46,  1.37s/it]INFO:backoff:Backing off request(...) for 11.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88227, Requested 3575. Please try again in 1.201s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 11.4 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 76 / 186  (40.9):  89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                     | 186/210 [04:27<00:31,  1.32s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88280, Requested 3568. Please try again in 1.232s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 78 / 189  (41.3):  90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 189/210 [04:32<00:27,  1.30s/it]INFO:backoff:Backing off request(...) for 1.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88280, Requested 3568. Please try again in 1.232s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.3 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 80 / 195  (41.0):  93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 195/210 [04:41<00:19,  1.28s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88293, Requested 3560. Please try again in 1.235s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 82 / 198  (41.4):  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 198/210 [04:45<00:17,  1.44s/it]INFO:backoff:Backing off request(...) for 1.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88251, Requested 3560. Please try again in 1.207s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 84 / 201  (41.8):  96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 201/210 [04:50<00:12,  1.42s/it]INFO:backoff:Backing off request(...) for 1.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88307, Requested 3560. Please try again in 1.244s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.4 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 84 / 204  (41.2):  97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 204/210 [04:54<00:08,  1.35s/it]INFO:backoff:Backing off request(...) for 2.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88285, Requested 3560. Please try again in 1.23s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.1 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 86 / 208  (41.3):  99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 208/210 [05:00<00:02,  1.41s/it]INFO:backoff:Backing off request(...) for 0.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88294, Requested 3559. Please try again in 1.235s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.1 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 88 / 210  (41.9): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [05:02<00:00,  1.44s/it]\n",
      "  2%|████▏                                                                                                                                                                                                                      | 4/210 [00:06<05:24,  1.57s/it]\n",
      "Average Metric: 1 / 3  (33.3):   1%|██▋                                                                                                                                                                                         | 3/210 [00:03<04:03,  1.18s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88368, Requested 3431. Please try again in 1.199s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 3 / 5  (60.0):   2%|████▍                                                                                                                                                                                       | 5/210 [00:06<05:11,  1.52s/it]INFO:backoff:Backing off request(...) for 0.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88371, Requested 3431. Please try again in 1.201s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.0 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 4 / 7  (57.1):   3%|██████▎                                                                                                                                                                                     | 7/210 [00:09<04:36,  1.36s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 87833, Requested 3431. Please try again in 842ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 5 / 12  (41.7):   6%|██████████▋                                                                                                                                                                               | 12/210 [00:16<04:41,  1.42s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88382, Requested 3413. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 6 / 16  (37.5):   8%|██████████████▏                                                                                                                                                                           | 16/210 [00:21<04:02,  1.25s/it]INFO:backoff:Backing off request(...) for 1.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88328, Requested 3459. Please try again in 1.191s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 8 / 20  (40.0):  10%|█████████████████▋                                                                                                                                                                        | 20/210 [00:26<03:48,  1.20s/it]INFO:backoff:Backing off request(...) for 1.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 87008, Requested 3459. Please try again in 311ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.0 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 10 / 22  (45.5):  10%|███████████████████▍                                                                                                                                                                     | 22/210 [00:29<04:06,  1.31s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88289, Requested 3451. Please try again in 1.16s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 11 / 25  (44.0):  12%|██████████████████████                                                                                                                                                                   | 25/210 [00:34<04:08,  1.34s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88355, Requested 3451. Please try again in 1.204s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 11 / 26  (42.3):  12%|██████████████████████▉                                                                                                                                                                  | 26/210 [00:35<04:07,  1.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 11 / 27  (40.7):  13%|███████████████████████▊                                                                                                                                                                 | 27/210 [00:37<04:35,  1.51s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88381, Requested 3405. Please try again in 1.19s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 12 / 30  (40.0):  14%|██████████████████████████▍                                                                                                                                                              | 30/210 [00:41<04:26,  1.48s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88394, Requested 3405. Please try again in 1.199s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 13 / 34  (38.2):  16%|█████████████████████████████▉                                                                                                                                                           | 34/210 [00:47<04:46,  1.63s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88331, Requested 3463. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 14 / 37  (37.8):  18%|████████████████████████████████▌                                                                                                                                                        | 37/210 [00:51<03:54,  1.36s/it]INFO:backoff:Backing off request(...) for 1.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88302, Requested 3463. Please try again in 1.176s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.5 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 14 / 40  (35.0):  19%|███████████████████████████████████▏                                                                                                                                                     | 40/210 [00:56<04:22,  1.55s/it]INFO:backoff:Backing off request(...) for 4.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88311, Requested 3463. Please try again in 1.182s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 4.0 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 17 / 45  (37.8):  21%|███████████████████████████████████████▋                                                                                                                                                 | 45/210 [01:02<03:40,  1.34s/it]INFO:backoff:Backing off request(...) for 2.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88287, Requested 3463. Please try again in 1.166s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 18 / 46  (39.1):  22%|████████████████████████████████████████▌                                                                                                                                                | 46/210 [01:03<03:41,  1.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.8 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 18 / 50  (36.0):  24%|████████████████████████████████████████████                                                                                                                                             | 50/210 [01:09<03:37,  1.36s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88350, Requested 3417. Please try again in 1.178s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 20 / 53  (37.7):  25%|██████████████████████████████████████████████▋                                                                                                                                          | 53/210 [01:14<03:42,  1.41s/it]INFO:backoff:Backing off request(...) for 1.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88361, Requested 3433. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 22 / 56  (39.3):  27%|█████████████████████████████████████████████████▎                                                                                                                                       | 56/210 [01:18<03:16,  1.28s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88328, Requested 3433. Please try again in 1.174s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 22 / 57  (38.6):  27%|██████████████████████████████████████████████████▏                                                                                                                                      | 57/210 [01:19<03:13,  1.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 23 / 59  (39.0):  28%|███████████████████████████████████████████████████▉                                                                                                                                     | 59/210 [01:22<03:21,  1.34s/it]INFO:backoff:Backing off request(...) for 2.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88317, Requested 3433. Please try again in 1.166s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 24 / 60  (40.0):  29%|████████████████████████████████████████████████████▊                                                                                                                                    | 60/210 [01:23<03:26,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.9 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 26 / 63  (41.3):  30%|███████████████████████████████████████████████████████▌                                                                                                                                 | 63/210 [01:28<03:53,  1.59s/it]INFO:backoff:Backing off request(...) for 0.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88389, Requested 3418. Please try again in 1.204s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.4 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 28 / 66  (42.4):  31%|██████████████████████████████████████████████████████████▏                                                                                                                              | 66/210 [01:32<03:23,  1.41s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88405, Requested 3423. Please try again in 1.218s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 28 / 67  (41.8):  32%|███████████████████████████████████████████████████████████                                                                                                                              | 67/210 [01:33<03:12,  1.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 29 / 69  (42.0):  33%|████████████████████████████████████████████████████████████▊                                                                                                                            | 69/210 [01:37<03:30,  1.50s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88332, Requested 3423. Please try again in 1.17s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 29 / 72  (40.3):  34%|███████████████████████████████████████████████████████████████▍                                                                                                                         | 72/210 [01:41<03:30,  1.52s/it]INFO:backoff:Backing off request(...) for 2.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88372, Requested 3423. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.6 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 31 / 76  (40.8):  36%|██████████████████████████████████████████████████████████████████▉                                                                                                                      | 76/210 [01:46<03:04,  1.38s/it]INFO:backoff:Backing off request(...) for 6.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88353, Requested 3423. Please try again in 1.184s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.3 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 32 / 83  (38.6):  40%|█████████████████████████████████████████████████████████████████████████                                                                                                                | 83/210 [01:57<03:28,  1.64s/it]INFO:backoff:Backing off request(...) for 8.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88302, Requested 3423. Please try again in 1.15s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 8.7 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 33 / 96  (34.4):  46%|████████████████████████████████████████████████████████████████████████████████████▌                                                                                                    | 96/210 [02:15<02:11,  1.16s/it]INFO:backoff:Backing off request(...) for 0.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88348, Requested 3435. Please try again in 1.188s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.1 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 33 / 99  (33.3):  47%|███████████████████████████████████████████████████████████████████████████████████████▏                                                                                                 | 99/210 [02:20<02:46,  1.50s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88373, Requested 3430. Please try again in 1.202s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 33 / 105  (31.4):  50%|███████████████████████████████████████████████████████████████████████████████████████████▌                                                                                           | 105/210 [02:28<02:22,  1.35s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88373, Requested 3421. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 37 / 110  (33.6):  52%|███████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                       | 110/210 [02:34<02:04,  1.24s/it]INFO:backoff:Backing off request(...) for 0.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 87020, Requested 3425. Please try again in 296ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.8 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 38 / 112  (33.9):  53%|█████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                                     | 112/210 [02:38<02:40,  1.64s/it]INFO:backoff:Backing off request(...) for 1.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88369, Requested 3425. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.9 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 39 / 115  (33.9):  55%|████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                  | 115/210 [02:42<02:14,  1.41s/it]INFO:backoff:Backing off request(...) for 1.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88327, Requested 3425. Please try again in 1.168s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.8 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 119  (34.5):  57%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                               | 119/210 [02:48<02:23,  1.57s/it]INFO:backoff:Backing off request(...) for 1.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88382, Requested 3425. Please try again in 1.204s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.8 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 42 / 122  (34.4):  58%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                            | 122/210 [02:52<02:10,  1.48s/it]INFO:backoff:Backing off request(...) for 3.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88359, Requested 3425. Please try again in 1.189s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.0 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 48 / 131  (36.6):  62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                    | 131/210 [03:05<01:47,  1.36s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88288, Requested 3428. Please try again in 1.144s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50 / 138  (36.2):  66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                              | 138/210 [03:15<01:41,  1.40s/it]INFO:backoff:Backing off request(...) for 0.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88391, Requested 3421. Please try again in 1.208s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.3 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 51 / 140  (36.4):  67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                             | 140/210 [03:18<01:33,  1.33s/it]INFO:backoff:Backing off request(...) for 1.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88327, Requested 3421. Please try again in 1.165s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.2 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 52 / 146  (35.6):  70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                       | 146/210 [03:26<01:25,  1.34s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88361, Requested 3430. Please try again in 1.194s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 52 / 148  (35.1):  70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                                                      | 148/210 [03:29<01:21,  1.31s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88361, Requested 3430. Please try again in 1.194s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 52 / 149  (34.9):  71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                     | 149/210 [03:30<01:14,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 54 / 151  (35.8):  72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                   | 151/210 [03:33<01:25,  1.44s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88365, Requested 3430. Please try again in 1.196s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 58 / 155  (37.4):  74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                | 155/210 [03:38<01:08,  1.25s/it]INFO:backoff:Backing off request(...) for 6.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 87039, Requested 3430. Please try again in 312ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.7 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 63 / 161  (39.1):  77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                          | 161/210 [03:46<01:04,  1.31s/it]INFO:backoff:Backing off request(...) for 15.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 89609, Requested 3430. Please try again in 2.026s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 15.1 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 68 / 175  (38.9):  83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                              | 175/210 [04:07<00:41,  1.18s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88323, Requested 3425. Please try again in 1.165s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 71 / 178  (39.9):  85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                            | 178/210 [04:12<00:39,  1.24s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88377, Requested 3410. Please try again in 1.191s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 72 / 181  (39.8):  86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                         | 181/210 [04:15<00:39,  1.35s/it]INFO:backoff:Backing off request(...) for 1.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88350, Requested 3410. Please try again in 1.173s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 73 / 184  (39.7):  88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                      | 184/210 [04:20<00:37,  1.43s/it]INFO:backoff:Backing off request(...) for 3.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88359, Requested 3410. Please try again in 1.179s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.6 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 77 / 189  (40.7):  90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 189/210 [04:28<00:28,  1.36s/it]INFO:backoff:Backing off request(...) for 0.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88352, Requested 3410. Please try again in 1.174s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 80 / 198  (40.4):  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 198/210 [04:40<00:15,  1.25s/it]INFO:backoff:Backing off request(...) for 0.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88356, Requested 3419. Please try again in 1.183s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.0 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 82 / 200  (41.0):  95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 200/210 [04:42<00:12,  1.27s/it]INFO:backoff:Backing off request(...) for 1.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88349, Requested 3419. Please try again in 1.178s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 83 / 204  (40.7):  97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 204/210 [04:49<00:09,  1.57s/it]INFO:backoff:Backing off request(...) for 3.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88378, Requested 3419. Please try again in 1.198s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 3.0 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 86 / 210  (41.0): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [04:57<00:00,  1.42s/it]\n",
      "  4%|████████▎                                                                                                                                                                                                                  | 8/210 [00:13<05:47,  1.72s/it]\n",
      "Average Metric: 3 / 8  (37.5):   4%|███████▏                                                                                                                                                                                    | 8/210 [00:09<04:06,  1.22s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88259, Requested 3679. Please try again in 1.292s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 4 / 10  (40.0):   5%|████████▊                                                                                                                                                                                 | 10/210 [00:12<04:36,  1.38s/it]INFO:backoff:Backing off request(...) for 1.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88294, Requested 3679. Please try again in 1.315s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.9 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 6 / 13  (46.2):   6%|███████████▌                                                                                                                                                                              | 13/210 [00:16<04:50,  1.47s/it]INFO:backoff:Backing off request(...) for 2.3s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88297, Requested 3679. Please try again in 1.317s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.3 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 8 / 17  (47.1):   8%|███████████████                                                                                                                                                                           | 17/210 [00:22<05:06,  1.59s/it]INFO:backoff:Backing off request(...) for 6.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88299, Requested 3679. Please try again in 1.318s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 9 / 18  (50.0):   9%|███████████████▉                                                                                                                                                                          | 18/210 [00:24<04:49,  1.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 6.9 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 11 / 24  (45.8):  11%|█████████████████████▏                                                                                                                                                                   | 24/210 [00:34<05:16,  1.70s/it]INFO:backoff:Backing off request(...) for 13.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88294, Requested 3679. Please try again in 1.315s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 13.8 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 17 / 39  (43.6):  19%|██████████████████████████████████▎                                                                                                                                                      | 39/210 [00:58<04:04,  1.43s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88363, Requested 3670. Please try again in 1.355s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 19 / 43  (44.2):  20%|█████████████████████████████████████                                                                                                                                                    | 42/210 [01:03<03:40,  1.32s/it]INFO:backoff:Backing off request(...) for 0.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88288, Requested 3679. Please try again in 1.311s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 20 / 44  (45.5):  20%|█████████████████████████████████████▉                                                                                                                                                   | 43/210 [01:13<03:46,  1.35s/it]IOStream.flush timed out\n",
      "IOStream.flush timed out\n",
      "INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86941, Requested 3679. Please try again in 413ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "IOStream.flush timed out\n",
      "Average Metric: 19 / 43  (44.2):  20%|█████████████████████████████████████▉                                                                                                                                                   | 43/210 [01:03<03:46,  1.35s/it]\n",
      "Average Metric: 28 / 56  (50.0):  26%|████████████████████████████████████████████████▍                                                                                                                                        | 55/210 [01:24<03:26,  1.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.7 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n",
      "Backing off 0.6 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 30 / 59  (50.8):  28%|███████████████████████████████████████████████████▉                                                                                                                                     | 59/210 [01:28<02:55,  1.16s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88316, Requested 3670. Please try again in 1.324s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 33 / 62  (53.2):  30%|██████████████████████████████████████████████████████▌                                                                                                                                  | 62/210 [01:33<03:18,  1.34s/it]INFO:backoff:Backing off request(...) for 1.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88275, Requested 3670. Please try again in 1.296s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 34 / 63  (54.0):  30%|███████████████████████████████████████████████████████▌                                                                                                                                 | 63/210 [01:34<03:13,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.2 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 35 / 66  (53.0):  31%|██████████████████████████████████████████████████████████▏                                                                                                                              | 66/210 [01:39<03:34,  1.49s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88266, Requested 3687. Please try again in 1.302s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 35 / 67  (52.2):  32%|███████████████████████████████████████████████████████████                                                                                                                              | 67/210 [01:40<03:29,  1.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 36 / 70  (51.4):  33%|█████████████████████████████████████████████████████████████▋                                                                                                                           | 70/210 [01:45<03:19,  1.42s/it]INFO:backoff:Backing off request(...) for 1.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 86931, Requested 3687. Please try again in 412ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.7 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 36 / 72  (50.0):  34%|███████████████████████████████████████████████████████████████▍                                                                                                                         | 72/210 [01:48<03:26,  1.50s/it]INFO:backoff:Backing off request(...) for 2.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88229, Requested 3687. Please try again in 1.277s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.8 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 37 / 76  (48.7):  36%|██████████████████████████████████████████████████████████████████▉                                                                                                                      | 76/210 [01:54<03:19,  1.49s/it]INFO:backoff:Backing off request(...) for 2.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88291, Requested 3687. Please try again in 1.318s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 2.8 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 40 / 80  (50.0):  38%|██████████████████████████████████████████████████████████████████████▍                                                                                                                  | 80/210 [02:02<03:58,  1.83s/it]INFO:backoff:Backing off request(...) for 14.7s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88273, Requested 3687. Please try again in 1.306s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 14.7 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 41 / 91  (45.1):  43%|████████████████████████████████████████████████████████████████████████████████▏                                                                                                        | 91/210 [02:19<03:06,  1.57s/it]INFO:backoff:Backing off request(...) for 19.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88306, Requested 3687. Please try again in 1.328s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 19.1 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 42 / 106  (39.6):  50%|████████████████████████████████████████████████████████████████████████████████████████████▎                                                                                          | 106/210 [02:41<02:38,  1.53s/it]INFO:backoff:Backing off request(...) for 1.1s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88305, Requested 3687. Please try again in 1.328s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.1 seconds after 7 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 45 / 109  (41.3):  52%|██████████████████████████████████████████████████████████████████████████████████████████████▉                                                                                        | 109/210 [02:46<02:31,  1.50s/it]INFO:backoff:Backing off request(...) for 81.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88298, Requested 3687. Please try again in 1.323s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 81.2 seconds after 8 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 65 / 163  (39.9):  78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                         | 163/210 [04:10<01:14,  1.58s/it]INFO:backoff:Backing off request(...) for 0.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88316, Requested 3656. Please try again in 1.314s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.5 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 66 / 166  (39.8):  79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                      | 166/210 [04:15<01:14,  1.70s/it]INFO:backoff:Backing off request(...) for 1.8s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88357, Requested 3656. Please try again in 1.342s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.8 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 68 / 170  (40.0):  81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 170/210 [04:21<01:05,  1.63s/it]INFO:backoff:Backing off request(...) for 0.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88313, Requested 3664. Please try again in 1.318s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.2 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 70 / 173  (40.5):  82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                | 173/210 [04:26<01:01,  1.67s/it]INFO:backoff:Backing off request(...) for 0.6s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88296, Requested 3679. Please try again in 1.316s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.6 seconds after 1 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 71 / 175  (40.6):  83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                              | 175/210 [04:30<00:56,  1.62s/it]INFO:backoff:Backing off request(...) for 1.5s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88295, Requested 3679. Please try again in 1.316s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.5 seconds after 2 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 74 / 178  (41.6):  85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                            | 178/210 [04:34<00:48,  1.52s/it]INFO:backoff:Backing off request(...) for 1.2s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88295, Requested 3679. Please try again in 1.316s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.2 seconds after 3 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 75 / 181  (41.4):  86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                         | 181/210 [04:38<00:44,  1.55s/it]INFO:backoff:Backing off request(...) for 1.0s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88303, Requested 3679. Please try again in 1.321s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 1.0 seconds after 4 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 76 / 184  (41.3):  88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                      | 184/210 [04:43<00:41,  1.60s/it]INFO:backoff:Backing off request(...) for 0.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88314, Requested 3679. Please try again in 1.328s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 0.9 seconds after 5 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 78 / 187  (41.7):  89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                    | 187/210 [04:48<00:34,  1.48s/it]INFO:backoff:Backing off request(...) for 21.9s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88253, Requested 3679. Please try again in 1.288s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 21.9 seconds after 6 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 85 / 203  (41.9):  97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉      | 203/210 [05:12<00:10,  1.44s/it]INFO:backoff:Backing off request(...) for 57.4s (openai.RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-3.5-turbo-instruct in organization org-DNIQnZOwEXH3HneN5ZlhWRmr on tokens per min (TPM): Limit 90000, Used 88264, Requested 3679. Please try again in 1.295s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Average Metric: 86 / 204  (42.2):  97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 204/210 [05:14<00:08,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backing off 57.4 seconds after 7 tries calling function <function GPT3.request at 0x12e9b4860> with kwargs {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 90 / 210  (42.9): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 210/210 [06:12<00:00,  1.78s/it]\n"
     ]
    }
   ],
   "source": [
    "from dspy.teleprompt import BootstrapFewShotWithRandomSearch, BootstrapFewShot\n",
    "\n",
    "# with dspy.context(lm=gpt3_turbo_instruct):\n",
    "dspy.settings.configure(lm=gpt3_turbo_instruct)\n",
    "\n",
    "config = dict(max_bootstrapped_demos=10, max_labeled_demos=30, num_candidate_programs=4, num_threads=2)\n",
    "optimiser = BootstrapFewShotWithRandomSearch(metric=grading_accuracy_metric, **config)\n",
    "# optimiser = BootstrapFewShot(metric=grading_accuracy, **config)\n",
    "optimised_program = optimiser.compile(asag_cot_pipeline, trainset=train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "ddeb690f-5648-4349-80ab-01a3927bd4bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prog = ChainOfThought(AssessmentSignature(question, student_answer, reference_answer -> assessment\n",
       "    instructions='\\n    Role: Academic Tutor\\n    Task: Make an assessment of the student\\'s answer by comparing it with the reference answer.\\n    You are essentially assessing whether the student\\'s answer was correct, or needs additional feedback.\\n    \\n    The value for assessment must be EXACTLY one of the following classes:\\n    \"correct\", \"partially_correct_incomplete\", \"contradictory\", \"non_domain\", \"irrelevant\"\\n    Include the underscores if the class contains it.\\n\\n    Definitions of the classes:\\n    - correct: if the student answer is a complete and correct paraphrase of the reference answer\\n    - partially_correct_incomplete: if it is a partially correct answer containing some but not\\n    all information from the reference answer\\n    - contradictory: if the student answer explicitly contradicts the reference answer\\n    - irrelevant:  if the student answer is talking about domain content but not providing the necessary information\\n    - non_domain: if the student utterance does not include domain content, e.g., “I don’t know”, “what the book says”, “you are stupid”\\n\\n    The \"question\" field doesn\\'t provide too much context apart from the final few words of the problem.\\n    So don\\'t focus on the question too much.\\n    '\n",
       "    question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
       "    student_answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Student Answer:', 'desc': '${student_answer}'})\n",
       "    reference_answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Reference Answer:', 'desc': '${reference_answer}'})\n",
       "    assessment = Field(annotation=str required=True json_schema_extra={'desc': 'the final assessment', '__dspy_field_type': 'output', 'prefix': 'Assessment:'})\n",
       "))"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimised_program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "f8f10fe6-9359-4fdb-9e07-67aeeaa0af21",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimised_program.save(\"asag_cot_better_instruction_prompt.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "93edcb32-42e8-457e-8b8c-a70ef737cc4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Role: Academic Tutor\n",
      "    Task: Make an assessment of the student's answer by comparing it with the reference answer.\n",
      "    You are essentially assessing whether the student's answer was correct, or needs additional feedback.\n",
      "    \n",
      "    The value for assessment must be EXACTLY one of the following classes:\n",
      "    \"correct\", \"partially_correct_incomplete\", \"contradictory\", \"non_domain\", \"irrelevant\"\n",
      "    Include the underscores if the class contains it.\n",
      "\n",
      "    Definitions of the classes:\n",
      "    - correct: if the student answer is a complete and correct paraphrase of the reference answer\n",
      "    - partially_correct_incomplete: if it is a partially correct answer containing some but not\n",
      "    all information from the reference answer\n",
      "    - contradictory: if the student answer explicitly contradicts the reference answer\n",
      "    - irrelevant:  if the student answer is talking about domain content but not providing the necessary information\n",
      "    - non_domain: if the student utterance does not include domain content, e.g., “I don’t know”, “what the book says”, “you are stupid”\n",
      "\n",
      "    The \"question\" field doesn't provide too much context apart from the final few words of the problem.\n",
      "    So don't focus on the question too much.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Question: ${question}\n",
      "\n",
      "Student Answer: ${student_answer}\n",
      "\n",
      "Reference Answer: ${reference_answer}\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the assessment}. We ...\n",
      "\n",
      "Assessment: the final assessment\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why was bulb A on when switch Y was open and switch Z was closed?\n",
      "\n",
      "Student Answer: the battery was contained in a circut\n",
      "\n",
      "Reference Answer: Bulb A is still contained in a closed path with the battery.\n",
      "\n",
      "Reasoning: Let's think step by step in order to produce the assessment. We can see that the student's answer is partially correct, as they mention the battery being contained in a circuit, but they do not mention the switch Y being open and switch Z being closed. Therefore, the student's answer is partially correct but incomplete.\n",
      "\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: When switch X was closed and switch Y was open, why was bulb A on?\n",
      "\n",
      "Student Answer: It has its own closed path\n",
      "\n",
      "Reference Answer: Bulb A was still contained in the same closed path with the battery.\n",
      "\n",
      "Reasoning: Let's think step by step in order to produce the assessment. We can see that the student's answer is partially correct, as it correctly identifies that bulb A is in its own closed path. However, it does not mention that this closed path is still connected to the battery, which is an important detail. Therefore, the assessment should be \"partially_correct_incomplete\".\n",
      "\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain your reasoning.\n",
      "\n",
      "Student Answer: i dont know\n",
      "\n",
      "Reference Answer: A and C are in different paths with the battery\n",
      "\n",
      "Reasoning: Let's think step by step in order to produce the assessment. We can see that the student answer is not providing any relevant information about the paths or the battery. Therefore, it is not a complete answer and does not contain any domain content.\n",
      "\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\n",
      "\n",
      "Student Answer: i dont know\n",
      "\n",
      "Reference Answer: Terminal 1 and the positive terminal are separated by the gap\n",
      "\n",
      "Reasoning: Let's think step by step in order to produce the assessment. We can see that the student answer does not provide any relevant information about the connection between terminal 1 and the positive terminal. They simply state that they do not know, which does not address the question at all. Therefore, the student answer falls under the non_domain class.\n",
      "\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: Under what circumstances will a switch affect a bulb?\n",
      "Student Answer: it will turn it off\n",
      "Reference Answer: When the switch and the bulb are contained in the same path\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Describe the paths in this diagram and explain how those paths account for the results.\n",
      "Student Answer: Bulb B and Bulb C are contained in a closed path.\n",
      "Reference Answer: Bulb A is in a path which does not contain B and C, so bulbs B and C don't affect it. Bulbs B and C are in the same path. They affect each other, but Bulb A doesn't affect them.\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.\n",
      "Student Answer: The positive battery terminal is seperated by a gap from termnial one.\n",
      "Reference Answer: Terminal 2 and the positive terminal are separated by the gap\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain your reasoning.\n",
      "Student Answer: tell answer\n",
      "Reference Answer: If bulb A burns out, B and C are no longer in a closed path with the battery\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: What do you think the red X means?\n",
      "Student Answer: don\"t do it.\n",
      "Reference Answer: the battery is damaged\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\n",
      "Student Answer: There is a difference in electrical states.\n",
      "Reference Answer: the bulb is damaged\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\n",
      "Student Answer: there terminals are separated by a gap when the voltage jumps from 0 to 1.5.\n",
      "Reference Answer: the bulb is damaged\n",
      "Assessment: correct\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 0 for terminal 1 and terminal 4.\n",
      "Student Answer: terminal 1 is connected to terminal 4.\n",
      "Reference Answer: Terminals 1 and 4 are connected\n",
      "Assessment: correct\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.\n",
      "Student Answer: there will be a gap between the positive battery terminal and the bulb terminal\n",
      "Reference Answer: Measuring voltage indicates the place where the electrical state changes due to a gap.\n",
      "Assessment: contradictory\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why circuit 4 is not a short circuit.\n",
      "Student Answer: Because neither of the components are connected to themselves.\n",
      "Reference Answer: Circuit 4 has no closed paths\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 0 for terminal 6 and the positive terminal.\n",
      "Student Answer: i dont know\n",
      "Reference Answer: Terminal 6 and the positive terminal are connected\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\n",
      "Student Answer: Because the meter was hooked up to two sides of the battery.\n",
      "Reference Answer: Terminal 1 and the positive terminal are not connected\n",
      "Assessment: contradictory\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.\n",
      "Student Answer: The positive battery terminal is separated by a gap from terminal 1. and germinal 3 is connected to the negative battery termina\n",
      "Reference Answer: Terminal 3 and the positive terminal are separated by the gap\n",
      "Assessment: correct\n",
      "\n",
      "---\n",
      "\n",
      "Question: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\n",
      "Student Answer: jdkjgkfjg\n",
      "Reference Answer: the bulb is damaged\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: When switch X was closed and switch Y was open, why was bulb A on?\n",
      "Student Answer: because it was contained in same path as the battery\n",
      "Reference Answer: Bulb A was still contained in the same closed path with the battery.\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why do you think those terminals and the negative battery terminal are in the same state?\n",
      "Student Answer: they are have 0 volts.\n",
      "Reference Answer: Terminals 1, 2 and 3 are connected to the negative battery terminal\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.\n",
      "Student Answer: The positive battery terminal is separated by a gap from terminal 1.\n",
      "Reference Answer: Terminal 2 and the positive terminal are not connected\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why?\n",
      "Student Answer: The voltage does not change.\n",
      "Reference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why do you think those terminals and the negative battery terminal are in the same state?\n",
      "Student Answer: Because bulb B is damaged and does not allow the positive battery current to reach the other side.\n",
      "Reference Answer: Terminals 1, 2 and 3 are connected to the negative battery terminal\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.\n",
      "Student Answer: they are seperated by a gap from bulb C\n",
      "Reference Answer: There is a gap between terminals 1 and 6\n",
      "Assessment: correct\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain your reasoning.\n",
      "Student Answer: njo\n",
      "Reference Answer: If bulb A burns out, B and C are no longer in a closed path with the battery\n",
      "Assessment: non_domain\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why?\n",
      "Student Answer: because a battery always has a charge in its positive terminal\n",
      "Reference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\n",
      "Assessment: contradictory\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.\n",
      "Student Answer: no gap\n",
      "Reference Answer: Terminal 5 and the positive terminal are connected\n",
      "Assessment: partially_correct_incomplete\n",
      "\n",
      "---\n",
      "\n",
      "Question: What role does the path play in determining whether or not a switch affects a bulb?\n",
      "Student Answer: bulb is contained in closed path\n",
      "Reference Answer: If a bulb and a switch are in the same path the switch affects the bulb\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: What is voltage?\n",
      "Student Answer: measure of the difference between the elecrical states of two terminals.\n",
      "Reference Answer: Voltage is the difference in electrical states between two terminals\n",
      "Assessment: correct\n",
      "\n",
      "---\n",
      "\n",
      "Question: Why?\n",
      "Student Answer: Because it has one regardless of the circuit\n",
      "Reference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\n",
      "Assessment: irrelevant\n",
      "\n",
      "---\n",
      "\n",
      "Question: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\n",
      "\n",
      "Student Answer: because voltage measures the difference between two points\n",
      "\n",
      "Reference Answer: Terminal 1 and the positive terminal are separated by the gap\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the assessment. We can see that the student answer is partially correct, as it correctly identifies that voltage measures the difference between two points. However, it does not mention the specific reason for the voltage reading of 1.5, which is the gap between the two terminals. Therefore, the assessment should be \"partially_correct_incomplete\".\n",
      "\n",
      "Assessment: partially_correct_incomplete\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n\\n\\nRole: Academic Tutor\\n    Task: Make an assessment of the student\\'s answer by comparing it with the reference answer.\\n    You are essentially assessing whether the student\\'s answer was correct, or needs additional feedback.\\n    \\n    The value for assessment must be EXACTLY one of the following classes:\\n    \"correct\", \"partially_correct_incomplete\", \"contradictory\", \"non_domain\", \"irrelevant\"\\n    Include the underscores if the class contains it.\\n\\n    Definitions of the classes:\\n    - correct: if the student answer is a complete and correct paraphrase of the reference answer\\n    - partially_correct_incomplete: if it is a partially correct answer containing some but not\\n    all information from the reference answer\\n    - contradictory: if the student answer explicitly contradicts the reference answer\\n    - irrelevant:  if the student answer is talking about domain content but not providing the necessary information\\n    - non_domain: if the student utterance does not include domain content, e.g., “I don’t know”, “what the book says”, “you are stupid”\\n\\n    The \"question\" field doesn\\'t provide too much context apart from the final few words of the problem.\\n    So don\\'t focus on the question too much.\\n\\n---\\n\\nFollow the following format.\\n\\nQuestion: ${question}\\n\\nStudent Answer: ${student_answer}\\n\\nReference Answer: ${reference_answer}\\n\\nReasoning: Let\\'s think step by step in order to ${produce the assessment}. We ...\\n\\nAssessment: the final assessment\\n\\n---\\n\\nQuestion: Why was bulb A on when switch Y was open and switch Z was closed?\\n\\nStudent Answer: the battery was contained in a circut\\n\\nReference Answer: Bulb A is still contained in a closed path with the battery.\\n\\nReasoning: Let\\'s think step by step in order to produce the assessment. We can see that the student\\'s answer is partially correct, as they mention the battery being contained in a circuit, but they do not mention the switch Y being open and switch Z being closed. Therefore, the student\\'s answer is partially correct but incomplete.\\n\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: When switch X was closed and switch Y was open, why was bulb A on?\\n\\nStudent Answer: It has its own closed path\\n\\nReference Answer: Bulb A was still contained in the same closed path with the battery.\\n\\nReasoning: Let\\'s think step by step in order to produce the assessment. We can see that the student\\'s answer is partially correct, as it correctly identifies that bulb A is in its own closed path. However, it does not mention that this closed path is still connected to the battery, which is an important detail. Therefore, the assessment should be \"partially_correct_incomplete\".\\n\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: Explain your reasoning.\\n\\nStudent Answer: i dont know\\n\\nReference Answer: A and C are in different paths with the battery\\n\\nReasoning: Let\\'s think step by step in order to produce the assessment. We can see that the student answer is not providing any relevant information about the paths or the battery. Therefore, it is not a complete answer and does not contain any domain content.\\n\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\\n\\nStudent Answer: i dont know\\n\\nReference Answer: Terminal 1 and the positive terminal are separated by the gap\\n\\nReasoning: Let\\'s think step by step in order to produce the assessment. We can see that the student answer does not provide any relevant information about the connection between terminal 1 and the positive terminal. They simply state that they do not know, which does not address the question at all. Therefore, the student answer falls under the non_domain class.\\n\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: Under what circumstances will a switch affect a bulb?\\nStudent Answer: it will turn it off\\nReference Answer: When the switch and the bulb are contained in the same path\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Describe the paths in this diagram and explain how those paths account for the results.\\nStudent Answer: Bulb B and Bulb C are contained in a closed path.\\nReference Answer: Bulb A is in a path which does not contain B and C, so bulbs B and C don\\'t affect it. Bulbs B and C are in the same path. They affect each other, but Bulb A doesn\\'t affect them.\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.\\nStudent Answer: The positive battery terminal is seperated by a gap from termnial one.\\nReference Answer: Terminal 2 and the positive terminal are separated by the gap\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Explain your reasoning.\\nStudent Answer: tell answer\\nReference Answer: If bulb A burns out, B and C are no longer in a closed path with the battery\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: What do you think the red X means?\\nStudent Answer: don\"t do it.\\nReference Answer: the battery is damaged\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\\nStudent Answer: There is a difference in electrical states.\\nReference Answer: the bulb is damaged\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\\nStudent Answer: there terminals are separated by a gap when the voltage jumps from 0 to 1.5.\\nReference Answer: the bulb is damaged\\nAssessment: correct\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 0 for terminal 1 and terminal 4.\\nStudent Answer: terminal 1 is connected to terminal 4.\\nReference Answer: Terminals 1 and 4 are connected\\nAssessment: correct\\n\\n---\\n\\nQuestion: Why does measuring voltage help you locate a burned out bulb? Try to answer in terms of electrical states, connections and/ or a gap.\\nStudent Answer: there will be a gap between the positive battery terminal and the bulb terminal\\nReference Answer: Measuring voltage indicates the place where the electrical state changes due to a gap.\\nAssessment: contradictory\\n\\n---\\n\\nQuestion: Explain why circuit 4 is not a short circuit.\\nStudent Answer: Because neither of the components are connected to themselves.\\nReference Answer: Circuit 4 has no closed paths\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 0 for terminal 6 and the positive terminal.\\nStudent Answer: i dont know\\nReference Answer: Terminal 6 and the positive terminal are connected\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\\nStudent Answer: Because the meter was hooked up to two sides of the battery.\\nReference Answer: Terminal 1 and the positive terminal are not connected\\nAssessment: contradictory\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 3 and the positive terminal.\\nStudent Answer: The positive battery terminal is separated by a gap from terminal 1. and germinal 3 is connected to the negative battery termina\\nReference Answer: Terminal 3 and the positive terminal are separated by the gap\\nAssessment: correct\\n\\n---\\n\\nQuestion: As you move the multimeter leads from one bulb terminal to the next, what does it mean when the voltage reading jumps from 0 to 1.5?\\nStudent Answer: jdkjgkfjg\\nReference Answer: the bulb is damaged\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: When switch X was closed and switch Y was open, why was bulb A on?\\nStudent Answer: because it was contained in same path as the battery\\nReference Answer: Bulb A was still contained in the same closed path with the battery.\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: Why do you think those terminals and the negative battery terminal are in the same state?\\nStudent Answer: they are have 0 volts.\\nReference Answer: Terminals 1, 2 and 3 are connected to the negative battery terminal\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 2 and the positive terminal.\\nStudent Answer: The positive battery terminal is separated by a gap from terminal 1.\\nReference Answer: Terminal 2 and the positive terminal are not connected\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Why?\\nStudent Answer: The voltage does not change.\\nReference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Why do you think those terminals and the negative battery terminal are in the same state?\\nStudent Answer: Because bulb B is damaged and does not allow the positive battery current to reach the other side.\\nReference Answer: Terminals 1, 2 and 3 are connected to the negative battery terminal\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 1 and terminal 6.\\nStudent Answer: they are seperated by a gap from bulb C\\nReference Answer: There is a gap between terminals 1 and 6\\nAssessment: correct\\n\\n---\\n\\nQuestion: Explain your reasoning.\\nStudent Answer: njo\\nReference Answer: If bulb A burns out, B and C are no longer in a closed path with the battery\\nAssessment: non_domain\\n\\n---\\n\\nQuestion: Why?\\nStudent Answer: because a battery always has a charge in its positive terminal\\nReference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\\nAssessment: contradictory\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.\\nStudent Answer: no gap\\nReference Answer: Terminal 5 and the positive terminal are connected\\nAssessment: partially_correct_incomplete\\n\\n---\\n\\nQuestion: What role does the path play in determining whether or not a switch affects a bulb?\\nStudent Answer: bulb is contained in closed path\\nReference Answer: If a bulb and a switch are in the same path the switch affects the bulb\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: What is voltage?\\nStudent Answer: measure of the difference between the elecrical states of two terminals.\\nReference Answer: Voltage is the difference in electrical states between two terminals\\nAssessment: correct\\n\\n---\\n\\nQuestion: Why?\\nStudent Answer: Because it has one regardless of the circuit\\nReference Answer: A battery uses a chemical reaction to maintain different electrical states at the terminals\\nAssessment: irrelevant\\n\\n---\\n\\nQuestion: Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.\\n\\nStudent Answer: because voltage measures the difference between two points\\n\\nReference Answer: Terminal 1 and the positive terminal are separated by the gap\\n\\nReasoning: Let\\'s think step by step in order to\\x1b[32m produce the assessment. We can see that the student answer is partially correct, as it correctly identifies that voltage measures the difference between two points. However, it does not mention the specific reason for the voltage reading of 1.5, which is the gap between the two terminals. Therefore, the assessment should be \"partially_correct_incomplete\".\\n\\nAssessment: partially_correct_incomplete\\x1b[0m\\n\\n\\n'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt3_turbo_instruct.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ba0afa6-eb18-459f-8fcd-630716e43a46",
   "metadata": {},
   "source": [
    "## Evaluation on Dev Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "b45dab94-429e-41f5-9348-91729a823587",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 33 / 90  (36.7): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [02:07<00:00,  1.42s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_11f93 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_11f93 td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_11f93_row0_col0, #T_11f93_row0_col1, #T_11f93_row0_col2, #T_11f93_row0_col3, #T_11f93_row0_col4, #T_11f93_row0_col5, #T_11f93_row0_col6, #T_11f93_row1_col0, #T_11f93_row1_col1, #T_11f93_row1_col2, #T_11f93_row1_col3, #T_11f93_row1_col4, #T_11f93_row1_col5, #T_11f93_row1_col6, #T_11f93_row2_col0, #T_11f93_row2_col1, #T_11f93_row2_col2, #T_11f93_row2_col3, #T_11f93_row2_col4, #T_11f93_row2_col5, #T_11f93_row2_col6, #T_11f93_row3_col0, #T_11f93_row3_col1, #T_11f93_row3_col2, #T_11f93_row3_col3, #T_11f93_row3_col4, #T_11f93_row3_col5, #T_11f93_row3_col6, #T_11f93_row4_col0, #T_11f93_row4_col1, #T_11f93_row4_col2, #T_11f93_row4_col3, #T_11f93_row4_col4, #T_11f93_row4_col5, #T_11f93_row4_col6 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_11f93\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_11f93_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_11f93_level0_col1\" class=\"col_heading level0 col1\" >student_answer</th>\n",
       "      <th id=\"T_11f93_level0_col2\" class=\"col_heading level0 col2\" >reference_answer</th>\n",
       "      <th id=\"T_11f93_level0_col3\" class=\"col_heading level0 col3\" >example_assessment</th>\n",
       "      <th id=\"T_11f93_level0_col4\" class=\"col_heading level0 col4\" >rationale</th>\n",
       "      <th id=\"T_11f93_level0_col5\" class=\"col_heading level0 col5\" >pred_assessment</th>\n",
       "      <th id=\"T_11f93_level0_col6\" class=\"col_heading level0 col6\" >grading_accuracy_metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_11f93_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_11f93_row0_col0\" class=\"data row0 col0\" >Why does a damaged bulb impact a circuit?</td>\n",
       "      <td id=\"T_11f93_row0_col1\" class=\"data row0 col1\" >a damaged bulb does not make a connecton so the circut is not complete</td>\n",
       "      <td id=\"T_11f93_row0_col2\" class=\"data row0 col2\" >a damaged bulb creates a gap</td>\n",
       "      <td id=\"T_11f93_row0_col3\" class=\"data row0 col3\" >correct</td>\n",
       "      <td id=\"T_11f93_row0_col4\" class=\"data row0 col4\" >produce the assessment. We can see that the student answer mentions a damaged bulb not making a connection, which is partially correct. However, it does...</td>\n",
       "      <td id=\"T_11f93_row0_col5\" class=\"data row0 col5\" >partially_correct_incomplete</td>\n",
       "      <td id=\"T_11f93_row0_col6\" class=\"data row0 col6\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_11f93_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_11f93_row1_col0\" class=\"data row1 col0\" >Explain your reasoning.</td>\n",
       "      <td id=\"T_11f93_row1_col1\" class=\"data row1 col1\" >bulb B is in a separate path than bulb A and bulb C.</td>\n",
       "      <td id=\"T_11f93_row1_col2\" class=\"data row1 col2\" >Bulbs A and C are still in closed paths with the battery</td>\n",
       "      <td id=\"T_11f93_row1_col3\" class=\"data row1 col3\" >correct</td>\n",
       "      <td id=\"T_11f93_row1_col4\" class=\"data row1 col4\" >produce the assessment. We can see that the student answer mentions bulb B being in a separate path, which is not the same as saying...</td>\n",
       "      <td id=\"T_11f93_row1_col5\" class=\"data row1 col5\" >partially_correct_incomplete</td>\n",
       "      <td id=\"T_11f93_row1_col6\" class=\"data row1 col6\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_11f93_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_11f93_row2_col0\" class=\"data row2 col0\" >Explain why you got a voltage reading of 1.5 for terminal 1 and the positive terminal.</td>\n",
       "      <td id=\"T_11f93_row2_col1\" class=\"data row2 col1\" >Terminal 1 was connected to the positive terminal.</td>\n",
       "      <td id=\"T_11f93_row2_col2\" class=\"data row2 col2\" >Terminal 1 and the positive terminal are separated by the gap</td>\n",
       "      <td id=\"T_11f93_row2_col3\" class=\"data row2 col3\" >contradictory</td>\n",
       "      <td id=\"T_11f93_row2_col4\" class=\"data row2 col4\" >produce the assessment. We can see that the student answer correctly identifies that terminal 1 was connected to the positive terminal, but it does not...</td>\n",
       "      <td id=\"T_11f93_row2_col5\" class=\"data row2 col5\" >partially_correct_incomplete</td>\n",
       "      <td id=\"T_11f93_row2_col6\" class=\"data row2 col6\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_11f93_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_11f93_row3_col0\" class=\"data row3 col0\" >Explain why you got a voltage reading of 0 for terminal 5 and the positive terminal.</td>\n",
       "      <td id=\"T_11f93_row3_col1\" class=\"data row3 col1\" >Because. It was on the wrong side.</td>\n",
       "      <td id=\"T_11f93_row3_col2\" class=\"data row3 col2\" >Terminal 5 and the positive terminal are connected</td>\n",
       "      <td id=\"T_11f93_row3_col3\" class=\"data row3 col3\" >contradictory</td>\n",
       "      <td id=\"T_11f93_row3_col4\" class=\"data row3 col4\" >produce the assessment. We can see that the student answer is not relevant to the question and does not provide any information about the connection...</td>\n",
       "      <td id=\"T_11f93_row3_col5\" class=\"data row3 col5\" >non_domain</td>\n",
       "      <td id=\"T_11f93_row3_col6\" class=\"data row3 col6\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_11f93_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_11f93_row4_col0\" class=\"data row4 col0\" >Under what circumstances will a switch affect a bulb?</td>\n",
       "      <td id=\"T_11f93_row4_col1\" class=\"data row4 col1\" >If it is on the path with it and the battery</td>\n",
       "      <td id=\"T_11f93_row4_col2\" class=\"data row4 col2\" >When the switch and the bulb are contained in the same path</td>\n",
       "      <td id=\"T_11f93_row4_col3\" class=\"data row4 col3\" >partially_correct_incomplete</td>\n",
       "      <td id=\"T_11f93_row4_col4\" class=\"data row4 col4\" >produce the assessment. We can see that the student answer is partially correct, as it mentions the role of the path and the battery in...</td>\n",
       "      <td id=\"T_11f93_row4_col5\" class=\"data row4 col5\" >partially_correct_incomplete</td>\n",
       "      <td id=\"T_11f93_row4_col6\" class=\"data row4 col6\" >✔️ [1]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1339913d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center;\n",
       "                    font-size: 16px;\n",
       "                    font-weight: bold;\n",
       "                    color: #555;\n",
       "                    margin: 10px 0;'>\n",
       "                    ... 85 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "36.67"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dspy.evaluate import Evaluate\n",
    "\n",
    "# Set up the evaluator, which can be re-used in your code.\n",
    "evaluator = Evaluate(devset=dev_data, num_threads=1, display_progress=True, display_table=5)\n",
    "\n",
    "# Launch evaluation.\n",
    "evaluator(optimised_program, metric=grading_accuracy_metric)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6111e164-3cb7-47d5-8d48-779dc44d0788",
   "metadata": {},
   "source": [
    "## Evaluation on Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "431c0155-0bdd-44cb-bf8e-8176fd2bfdfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 97 / 296  (32.8):  20%|████████████████████████████████████████▌                                                                                                                                                                   | 296/1489 [08:10<33:18,  1.68s/it]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[92], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m evaluator \u001b[38;5;241m=\u001b[39m Evaluate(devset\u001b[38;5;241m=\u001b[39mdspy_test_dataset, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, display_table\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m# Launch evaluation.\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[43mevaluator\u001b[49m\u001b[43m(\u001b[49m\u001b[43moptimised_program\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetric\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrading_accuracy_metric\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/evaluate/evaluate.py:158\u001b[0m, in \u001b[0;36mEvaluate.__call__\u001b[0;34m(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)\u001b[0m\n\u001b[1;32m    155\u001b[0m tqdm\u001b[38;5;241m.\u001b[39mtqdm\u001b[38;5;241m.\u001b[39m_instances\u001b[38;5;241m.\u001b[39mclear()\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m num_threads \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m--> 158\u001b[0m     reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_single_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwrapped_program\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdisplay_progress\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    159\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    160\u001b[0m     reordered_devset, ncorrect, ntotal \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute_multi_thread(\n\u001b[1;32m    161\u001b[0m         wrapped_program,\n\u001b[1;32m    162\u001b[0m         devset,\n\u001b[1;32m    163\u001b[0m         num_threads,\n\u001b[1;32m    164\u001b[0m         display_progress,\n\u001b[1;32m    165\u001b[0m     )\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/evaluate/evaluate.py:67\u001b[0m, in \u001b[0;36mEvaluate._execute_single_thread\u001b[0;34m(self, wrapped_program, devset, display_progress)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, arg \u001b[38;5;129;01min\u001b[39;00m devset:\n\u001b[1;32m     66\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m logging_redirect_tqdm():\n\u001b[0;32m---> 67\u001b[0m         example_idx, example, prediction, score \u001b[38;5;241m=\u001b[39m \u001b[43mwrapped_program\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     68\u001b[0m         reordered_devset\u001b[38;5;241m.\u001b[39mappend((example_idx, example, prediction, score))\n\u001b[1;32m     69\u001b[0m         ncorrect \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m score\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/evaluate/evaluate.py:127\u001b[0m, in \u001b[0;36mEvaluate.__call__.<locals>.wrapped_program\u001b[0;34m(example_idx, example)\u001b[0m\n\u001b[1;32m    124\u001b[0m     thread_stacks[threading\u001b[38;5;241m.\u001b[39mget_ident()] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(dspy\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mmain_stack)\n\u001b[1;32m    126\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 127\u001b[0m     prediction \u001b[38;5;241m=\u001b[39m \u001b[43mprogram\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mexample\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    128\u001b[0m     score \u001b[38;5;241m=\u001b[39m metric(\n\u001b[1;32m    129\u001b[0m         example,\n\u001b[1;32m    130\u001b[0m         prediction,\n\u001b[1;32m    131\u001b[0m     )  \u001b[38;5;66;03m# FIXME: TODO: What's the right order? Maybe force name-based kwargs!\u001b[39;00m\n\u001b[1;32m    133\u001b[0m     \u001b[38;5;66;03m# increment assert and suggest failures to program's attributes\u001b[39;00m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/primitives/program.py:26\u001b[0m, in \u001b[0;36mModule.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 26\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[70], line 37\u001b[0m, in \u001b[0;36mASAGCoT.forward\u001b[0;34m(self, question, student_answer, reference_answer)\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, question: \u001b[38;5;28mstr\u001b[39m, student_answer: \u001b[38;5;28mstr\u001b[39m, reference_answer: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m---> 37\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprog\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquestion\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquestion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstudent_answer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstudent_answer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreference_answer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreference_answer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     39\u001b[0m     output\u001b[38;5;241m.\u001b[39massessment \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(output\u001b[38;5;241m.\u001b[39massessment)\u001b[38;5;241m.\u001b[39mlower()\n\u001b[1;32m     40\u001b[0m     dspy\u001b[38;5;241m.\u001b[39mSuggest(\n\u001b[1;32m     41\u001b[0m         output\u001b[38;5;241m.\u001b[39massessment \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcorrect\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpartially_correct_incomplete\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontradictory\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon_domain\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mirrelevant\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m     42\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00moutput\u001b[38;5;241m.\u001b[39massessment\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m must be exactly one of \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpartially_correct_incomplete\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontradictory\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnon_domain\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mirrelevant\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     43\u001b[0m     )\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/predict/predict.py:61\u001b[0m, in \u001b[0;36mPredict.__call__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 61\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/predict/chain_of_thought.py:59\u001b[0m, in \u001b[0;36mChainOfThought.forward\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m     57\u001b[0m     signature \u001b[38;5;241m=\u001b[39m new_signature\n\u001b[1;32m     58\u001b[0m     \u001b[38;5;66;03m# template = dsp.Template(self.signature.instructions, **new_signature)\u001b[39;00m\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dspy/predict/predict.py:103\u001b[0m, in \u001b[0;36mPredict.forward\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    100\u001b[0m template \u001b[38;5;241m=\u001b[39m signature_to_template(signature)\n\u001b[1;32m    102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 103\u001b[0m     x, C \u001b[38;5;241m=\u001b[39m \u001b[43mdsp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtemplate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstage\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    104\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;66;03m# Note: query_only=True means the instructions and examples are not included.\u001b[39;00m\n\u001b[1;32m    106\u001b[0m     \u001b[38;5;66;03m# I'm not really sure why we'd want to do that, but it's there.\u001b[39;00m\n\u001b[1;32m    107\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m dsp\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mcontext(lm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm, query_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/primitives/predict.py:77\u001b[0m, in \u001b[0;36m_generate.<locals>.do_generate\u001b[0;34m(example, stage, max_depth, original_example)\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[38;5;66;03m# Generate and extract the fields.\u001b[39;00m\n\u001b[1;32m     76\u001b[0m prompt \u001b[38;5;241m=\u001b[39m template(example)\n\u001b[0;32m---> 77\u001b[0m completions: \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any]] \u001b[38;5;241m=\u001b[39m \u001b[43mgenerator\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     78\u001b[0m completions: \u001b[38;5;28mlist\u001b[39m[Example] \u001b[38;5;241m=\u001b[39m [template\u001b[38;5;241m.\u001b[39mextract(example, p) \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m completions]\n\u001b[1;32m     80\u001b[0m \u001b[38;5;66;03m# Find the completions that are most complete.\u001b[39;00m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:178\u001b[0m, in \u001b[0;36mGPT3.__call__\u001b[0;34m(self, prompt, only_completed, return_sorted, **kwargs)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m return_sorted \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfor now\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    172\u001b[0m \u001b[38;5;66;03m# if kwargs.get(\"n\", 1) > 1:\u001b[39;00m\n\u001b[1;32m    173\u001b[0m \u001b[38;5;66;03m#     if self.model_type == \"chat\":\u001b[39;00m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;66;03m#         kwargs = {**kwargs}\u001b[39;00m\n\u001b[1;32m    175\u001b[0m \u001b[38;5;66;03m#     else:\u001b[39;00m\n\u001b[1;32m    176\u001b[0m \u001b[38;5;66;03m#         kwargs = {**kwargs, \"logprobs\": 5}\u001b[39;00m\n\u001b[0;32m--> 178\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    180\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog_usage(response)\n\u001b[1;32m    181\u001b[0m choices \u001b[38;5;241m=\u001b[39m response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/backoff/_sync.py:105\u001b[0m, in \u001b[0;36mretry_exception.<locals>.retry\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     96\u001b[0m details \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtarget\u001b[39m\u001b[38;5;124m\"\u001b[39m: target,\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124margs\u001b[39m\u001b[38;5;124m\"\u001b[39m: args,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124melapsed\u001b[39m\u001b[38;5;124m\"\u001b[39m: elapsed,\n\u001b[1;32m    102\u001b[0m }\n\u001b[1;32m    104\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 105\u001b[0m     ret \u001b[38;5;241m=\u001b[39m \u001b[43mtarget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m exception \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    107\u001b[0m     max_tries_exceeded \u001b[38;5;241m=\u001b[39m (tries \u001b[38;5;241m==\u001b[39m max_tries_value)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:144\u001b[0m, in \u001b[0;36mGPT3.request\u001b[0;34m(self, prompt, **kwargs)\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel_type\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs:\n\u001b[1;32m    142\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel_type\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m--> 144\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbasic_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:121\u001b[0m, in \u001b[0;36mGPT3.basic_request\u001b[0;34m(self, prompt, **kwargs)\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    120\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m prompt\n\u001b[0;32m--> 121\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mcompletions_request\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    123\u001b[0m history \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    124\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m: prompt,\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresponse\u001b[39m\u001b[38;5;124m\"\u001b[39m: response,\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkwargs\u001b[39m\u001b[38;5;124m\"\u001b[39m: kwargs,\n\u001b[1;32m    127\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw_kwargs\u001b[39m\u001b[38;5;124m\"\u001b[39m: raw_kwargs,\n\u001b[1;32m    128\u001b[0m }\n\u001b[1;32m    129\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhistory\u001b[38;5;241m.\u001b[39mappend(history)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:270\u001b[0m, in \u001b[0;36mcompletions_request\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m    267\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m OPENAI_LEGACY:\n\u001b[1;32m    268\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m cached_gpt3_request_v2_wrapped(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m--> 270\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mv1_cached_gpt3_request_v2_wrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mmodel_dump()\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/cache_utils.py:16\u001b[0m, in \u001b[0;36mnoop_decorator.<locals>.decorator.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(func)\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 16\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:243\u001b[0m, in \u001b[0;36mv1_cached_gpt3_request_v2_wrapped\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mlru_cache(maxsize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mif\u001b[39;00m cache_turn_on \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    241\u001b[0m \u001b[38;5;129m@NotebookCacheMemory\u001b[39m\u001b[38;5;241m.\u001b[39mcache\n\u001b[1;32m    242\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mv1_cached_gpt3_request_v2_wrapped\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mv1_cached_gpt3_request_v2\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/joblib/memory.py:655\u001b[0m, in \u001b[0;36mMemorizedFunc.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    654\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 655\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cached_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/joblib/memory.py:598\u001b[0m, in \u001b[0;36mMemorizedFunc._cached_call\u001b[0;34m(self, args, kwargs, shelving)\u001b[0m\n\u001b[1;32m    595\u001b[0m     must_call \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    597\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m must_call:\n\u001b[0;32m--> 598\u001b[0m     out, metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    599\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmmap_mode \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    600\u001b[0m         \u001b[38;5;66;03m# Memmap the output at the first call to be consistent with\u001b[39;00m\n\u001b[1;32m    601\u001b[0m         \u001b[38;5;66;03m# later calls\u001b[39;00m\n\u001b[1;32m    602\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verbose:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/joblib/memory.py:856\u001b[0m, in \u001b[0;36mMemorizedFunc.call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    854\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verbose \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m    855\u001b[0m     \u001b[38;5;28mprint\u001b[39m(format_call(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunc, args, kwargs))\n\u001b[0;32m--> 856\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    857\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_backend\u001b[38;5;241m.\u001b[39mdump_item(\n\u001b[1;32m    858\u001b[0m     [func_id, args_id], output, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verbose)\n\u001b[1;32m    860\u001b[0m duration \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m-\u001b[39m start_time\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/dsp/modules/gpt3.py:237\u001b[0m, in \u001b[0;36mv1_cached_gpt3_request_v2\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;129m@CacheMemory\u001b[39m\u001b[38;5;241m.\u001b[39mcache\n\u001b[1;32m    236\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mv1_cached_gpt3_request_v2\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 237\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompletions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_utils/_utils.py:277\u001b[0m, in \u001b[0;36mrequired_args.<locals>.inner.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    275\u001b[0m             msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required argument: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mquote(missing[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[0;32m--> 277\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/resources/completions.py:528\u001b[0m, in \u001b[0;36mCompletions.create\u001b[0;34m(self, model, prompt, best_of, echo, frequency_penalty, logit_bias, logprobs, max_tokens, n, presence_penalty, seed, stop, stream, stream_options, suffix, temperature, top_p, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m    499\u001b[0m \u001b[38;5;129m@required_args\u001b[39m([\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m], [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate\u001b[39m(\n\u001b[1;32m    501\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    526\u001b[0m     timeout: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m|\u001b[39m httpx\u001b[38;5;241m.\u001b[39mTimeout \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m|\u001b[39m NotGiven \u001b[38;5;241m=\u001b[39m NOT_GIVEN,\n\u001b[1;32m    527\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Completion \u001b[38;5;241m|\u001b[39m Stream[Completion]:\n\u001b[0;32m--> 528\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_post\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    529\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/completions\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    530\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmaybe_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    531\u001b[0m \u001b[43m            \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m    532\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmodel\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    533\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprompt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    534\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mbest_of\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mbest_of\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mecho\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mecho\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfrequency_penalty\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrequency_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    537\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlogit_bias\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogit_bias\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    538\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlogprobs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogprobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    539\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmax_tokens\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    540\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mn\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    541\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpresence_penalty\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mpresence_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    542\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mseed\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    543\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstop\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstop\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    544\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstream\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    545\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstream_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    546\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msuffix\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msuffix\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    547\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtemperature\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    548\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtop_p\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_p\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    549\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muser\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43muser\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    550\u001b[0m \u001b[43m            \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    551\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcompletion_create_params\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCompletionCreateParams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    552\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    553\u001b[0m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmake_request_options\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    554\u001b[0m \u001b[43m            \u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_headers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_body\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\n\u001b[1;32m    555\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCompletion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    557\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    558\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mStream\u001b[49m\u001b[43m[\u001b[49m\u001b[43mCompletion\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    559\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_base_client.py:1240\u001b[0m, in \u001b[0;36mSyncAPIClient.post\u001b[0;34m(self, path, cast_to, body, options, files, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1226\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\n\u001b[1;32m   1227\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1228\u001b[0m     path: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1235\u001b[0m     stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1236\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[1;32m   1237\u001b[0m     opts \u001b[38;5;241m=\u001b[39m FinalRequestOptions\u001b[38;5;241m.\u001b[39mconstruct(\n\u001b[1;32m   1238\u001b[0m         method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpost\u001b[39m\u001b[38;5;124m\"\u001b[39m, url\u001b[38;5;241m=\u001b[39mpath, json_data\u001b[38;5;241m=\u001b[39mbody, files\u001b[38;5;241m=\u001b[39mto_httpx_files(files), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions\n\u001b[1;32m   1239\u001b[0m     )\n\u001b[0;32m-> 1240\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m cast(ResponseT, \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_base_client.py:921\u001b[0m, in \u001b[0;36mSyncAPIClient.request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m    912\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    913\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    914\u001b[0m     cast_to: Type[ResponseT],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    919\u001b[0m     stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    920\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[0;32m--> 921\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    922\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    923\u001b[0m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    924\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    925\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    926\u001b[0m \u001b[43m        \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    927\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_base_client.py:1005\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1003\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retries \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_should_retry(err\u001b[38;5;241m.\u001b[39mresponse):\n\u001b[1;32m   1004\u001b[0m     err\u001b[38;5;241m.\u001b[39mresponse\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m-> 1005\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1006\u001b[0m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1007\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1008\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1009\u001b[0m \u001b[43m        \u001b[49m\u001b[43merr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1010\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1011\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1012\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1014\u001b[0m \u001b[38;5;66;03m# If the response is streamed then we need to explicitly read the response\u001b[39;00m\n\u001b[1;32m   1015\u001b[0m \u001b[38;5;66;03m# to completion before attempting to access the response text.\u001b[39;00m\n\u001b[1;32m   1016\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m err\u001b[38;5;241m.\u001b[39mresponse\u001b[38;5;241m.\u001b[39mis_closed:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_base_client.py:1053\u001b[0m, in \u001b[0;36mSyncAPIClient._retry_request\u001b[0;34m(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1049\u001b[0m \u001b[38;5;66;03m# In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\u001b[39;00m\n\u001b[1;32m   1050\u001b[0m \u001b[38;5;66;03m# different thread if necessary.\u001b[39;00m\n\u001b[1;32m   1051\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(timeout)\n\u001b[0;32m-> 1053\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1054\u001b[0m \u001b[43m    \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1055\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1056\u001b[0m \u001b[43m    \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1057\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1058\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1059\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/openai/_base_client.py:952\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m    949\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSending HTTP Request: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, request\u001b[38;5;241m.\u001b[39mmethod, request\u001b[38;5;241m.\u001b[39murl)\n\u001b[1;32m    951\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 952\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    953\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    954\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_should_stream_response_body\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    955\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    956\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    957\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mTimeoutException \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m    958\u001b[0m     log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered httpx.TimeoutException\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpx/_client.py:914\u001b[0m, in \u001b[0;36mClient.send\u001b[0;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[1;32m    906\u001b[0m follow_redirects \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    907\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfollow_redirects\n\u001b[1;32m    908\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(follow_redirects, UseClientDefault)\n\u001b[1;32m    909\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m follow_redirects\n\u001b[1;32m    910\u001b[0m )\n\u001b[1;32m    912\u001b[0m auth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_request_auth(request, auth)\n\u001b[0;32m--> 914\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_auth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    915\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    916\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    917\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    918\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    919\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    920\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    921\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m stream:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpx/_client.py:942\u001b[0m, in \u001b[0;36mClient._send_handling_auth\u001b[0;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[1;32m    939\u001b[0m request \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(auth_flow)\n\u001b[1;32m    941\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 942\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_handling_redirects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    943\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    944\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    945\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhistory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhistory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    946\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    947\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    948\u001b[0m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpx/_client.py:979\u001b[0m, in \u001b[0;36mClient._send_handling_redirects\u001b[0;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[1;32m    976\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_event_hooks[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m    977\u001b[0m     hook(request)\n\u001b[0;32m--> 979\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_single_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    980\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    981\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_event_hooks[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresponse\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpx/_client.py:1015\u001b[0m, in \u001b[0;36mClient._send_single_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m   1010\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m   1011\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAttempted to send an async request with a sync Client instance.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1012\u001b[0m     )\n\u001b[1;32m   1014\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request\u001b[38;5;241m=\u001b[39mrequest):\n\u001b[0;32m-> 1015\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mtransport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1017\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, SyncByteStream)\n\u001b[1;32m   1019\u001b[0m response\u001b[38;5;241m.\u001b[39mrequest \u001b[38;5;241m=\u001b[39m request\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpx/_transports/default.py:233\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    220\u001b[0m req \u001b[38;5;241m=\u001b[39m httpcore\u001b[38;5;241m.\u001b[39mRequest(\n\u001b[1;32m    221\u001b[0m     method\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[1;32m    222\u001b[0m     url\u001b[38;5;241m=\u001b[39mhttpcore\u001b[38;5;241m.\u001b[39mURL(\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    230\u001b[0m     extensions\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mextensions,\n\u001b[1;32m    231\u001b[0m )\n\u001b[1;32m    232\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 233\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mIterable)\n\u001b[1;32m    237\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m Response(\n\u001b[1;32m    238\u001b[0m     status_code\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mstatus,\n\u001b[1;32m    239\u001b[0m     headers\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m    240\u001b[0m     stream\u001b[38;5;241m=\u001b[39mResponseStream(resp\u001b[38;5;241m.\u001b[39mstream),\n\u001b[1;32m    241\u001b[0m     extensions\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mextensions,\n\u001b[1;32m    242\u001b[0m )\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py:216\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    213\u001b[0m         closing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_assign_requests_to_connections()\n\u001b[1;32m    215\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_connections(closing)\n\u001b[0;32m--> 216\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    218\u001b[0m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[1;32m    219\u001b[0m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n\u001b[1;32m    220\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, Iterable)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py:196\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    192\u001b[0m connection \u001b[38;5;241m=\u001b[39m pool_request\u001b[38;5;241m.\u001b[39mwait_for_connection(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m    194\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    195\u001b[0m     \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[0;32m--> 196\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpool_request\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\n\u001b[1;32m    198\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[1;32m    200\u001b[0m     \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[1;32m    202\u001b[0m     \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m    203\u001b[0m     \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n\u001b[1;32m    204\u001b[0m     pool_request\u001b[38;5;241m.\u001b[39mclear_connection()\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/connection.py:101\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connect_failed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m     99\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[0;32m--> 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_connection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/http11.py:143\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    141\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m Trace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresponse_closed\u001b[39m\u001b[38;5;124m\"\u001b[39m, logger, request) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[1;32m    142\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_response_closed()\n\u001b[0;32m--> 143\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/http11.py:113\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[1;32m    104\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreceive_response_headers\u001b[39m\u001b[38;5;124m\"\u001b[39m, logger, request, kwargs\n\u001b[1;32m    106\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[1;32m    107\u001b[0m     (\n\u001b[1;32m    108\u001b[0m         http_version,\n\u001b[1;32m    109\u001b[0m         status,\n\u001b[1;32m    110\u001b[0m         reason_phrase,\n\u001b[1;32m    111\u001b[0m         headers,\n\u001b[1;32m    112\u001b[0m         trailing_data,\n\u001b[0;32m--> 113\u001b[0m     ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_receive_response_headers\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    114\u001b[0m     trace\u001b[38;5;241m.\u001b[39mreturn_value \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    115\u001b[0m         http_version,\n\u001b[1;32m    116\u001b[0m         status,\n\u001b[1;32m    117\u001b[0m         reason_phrase,\n\u001b[1;32m    118\u001b[0m         headers,\n\u001b[1;32m    119\u001b[0m     )\n\u001b[1;32m    121\u001b[0m network_stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_network_stream\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/http11.py:186\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_response_headers\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    183\u001b[0m timeout \u001b[38;5;241m=\u001b[39m timeouts\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mread\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    185\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 186\u001b[0m     event \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_receive_event\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    187\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(event, h11\u001b[38;5;241m.\u001b[39mResponse):\n\u001b[1;32m    188\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_sync/http11.py:224\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_event\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    221\u001b[0m     event \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_h11_state\u001b[38;5;241m.\u001b[39mnext_event()\n\u001b[1;32m    223\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m event \u001b[38;5;129;01mis\u001b[39;00m h11\u001b[38;5;241m.\u001b[39mNEED_DATA:\n\u001b[0;32m--> 224\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_network_stream\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    225\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mREAD_NUM_BYTES\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\n\u001b[1;32m    226\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    228\u001b[0m     \u001b[38;5;66;03m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m    230\u001b[0m     \u001b[38;5;66;03m#     httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    234\u001b[0m     \u001b[38;5;66;03m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;66;03m# it as a ConnectError.\u001b[39;00m\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data \u001b[38;5;241m==\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_h11_state\u001b[38;5;241m.\u001b[39mtheir_state \u001b[38;5;241m==\u001b[39m h11\u001b[38;5;241m.\u001b[39mSEND_RESPONSE:\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/asag-experiment-Y6_GzHsZ-py3.11/lib/python3.11/site-packages/httpcore/_backends/sync.py:126\u001b[0m, in \u001b[0;36mSyncStream.read\u001b[0;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_exceptions(exc_map):\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sock\u001b[38;5;241m.\u001b[39msettimeout(timeout)\n\u001b[0;32m--> 126\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmax_bytes\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.2/lib/python3.11/ssl.py:1263\u001b[0m, in \u001b[0;36mSSLSocket.recv\u001b[0;34m(self, buflen, flags)\u001b[0m\n\u001b[1;32m   1259\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1260\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1261\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1262\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1263\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuflen\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1264\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1265\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv(buflen, flags)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.2/lib/python3.11/ssl.py:1136\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1134\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m, buffer)\n\u001b[1;32m   1135\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1136\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1137\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SSLError \u001b[38;5;28;01mas\u001b[39;00m x:\n\u001b[1;32m   1138\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m x\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m==\u001b[39m SSL_ERROR_EOF \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msuppress_ragged_eofs:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "evaluator = Evaluate(devset=dspy_test_dataset, num_threads=1, display_progress=True, display_table=5)\n",
    "\n",
    "# Launch evaluation.\n",
    "evaluator(optimised_program, metric=grading_accuracy_metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33cff67a-9d38-45c4-83e3-4bed9895b973",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f78d414c-ecca-4622-b76f-eecadf35731a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d5f7b38-a676-4d8d-b24e-8b3d57d27388",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c11449a-3c52-4272-a985-ea7059c3d526",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6f536fb-07cd-4a26-a025-0c333b40b344",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4b29bcf-a5da-497c-b3d3-bae63e4ba373",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8d9c84e-8767-4b14-93a9-8094228a0955",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f465ad7d-e106-49a8-9a7f-37cf422cac5b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}