create-conllu-troubleshoot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=5\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "# Define the language code, used in the file names\n",
    "lang_code = \"UA\"\n",
    "\n",
    "# Main path\n",
    "main_path = \"/home/tajak/Parlamint-translation\"\n",
    "\n",
    "# Define the translation model to be used\n",
    "opus_lang_code = \"sla\"\n",
    "\n",
    "# Check whether the path to the folder with conllu files is ok\n",
    "path = \"{}/Source-data/ParlaMint-{}.conllu/ParlaMint-{}.conllu\".format(main_path, lang_code, lang_code)\n",
    "\n",
    "# Define other paths\n",
    "extracted_dataframe_path = \"{}/results/{}/ParlaMint-{}-extracted-source-data.csv\".format(main_path, lang_code, lang_code)\n",
    "\n",
    "translated_dataframe_path = \"{}/results/{}/ParlaMint-{}-translated.csv\".format(main_path, lang_code, lang_code)\n",
    "translated_tokenized_dataframe_path = \"{}/results/{}/ParlaMint-{}-translated-tokenized.csv\".format(main_path,lang_code, lang_code)\n",
    "final_dataframe = \"{}/results/{}/ParlaMint-{}-final-dataframe.csv\".format(main_path,lang_code, lang_code)\n",
    "\n",
    "final_path = \"{}/Final-data/ParlaMint-{}.conllu/ParlaMint-{}.conllu\".format(main_path, lang_code, lang_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\t# Create a new conllu file with the updated information\\n\\t# do not save - for troubleshooting\\n\\n\\tos.makedirs(os.path.dirname(target_path), exist_ok=True)\\n\\tfinal_file = open(\"{}\".format(target_path), \"w\")\\n\\n\\tfor sentence in sentences:\\n\\t\\tfinal_file.write(sentence.serialize())\\n\\t\\n\\tfinal_file.close()\\n\\n\\tprint(\"Final file {} is saved.\".format(target_path))\\n\\n'"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_conllu(file, lang_code, main_path, final_dataframe, nlp):\n",
    "\t\"\"\"\n",
    "\tThe function takes the dataframe (df), created in previous steps and takes only the instances from the df that belong\n",
    "\tto the file that is in the argument. It linguistically processes the translated sentences from the file and saves the file.\n",
    "\tThen we add additional information (metadata and NER annotations) to it with the conllu parser and save the final conllu file.\n",
    "\n",
    "\tArgs:\n",
    "\t\t- file (str): file name from the files list (see above)\n",
    "\t\t- lang_code (str): the lang code that is used in the names of the files, it should be the same as for extract_text()\n",
    "\t\"\"\"\n",
    "\n",
    "\t# Process all sentences in the dataframe and save them to a conllu file\n",
    "\tfrom stanza.utils.conll import CoNLL\n",
    "\timport stanza\n",
    "\tfrom conllu import parse\n",
    "\timport ast\n",
    "\timport regex as re\n",
    "\timport os\n",
    "\timport pandas as pd\n",
    "\n",
    "\t# Use the dataframe, created in previous steps\n",
    "\tdf = pd.read_csv(\"{}\".format(final_dataframe), sep=\"\\t\", index_col = 0, na_filter = False)\n",
    "\n",
    "\t# Filter out only instances from the file in question\n",
    "\tdf = df[df[\"file\"] == file]\n",
    "\n",
    "\t# Add information on the target path\n",
    "\tdf[\"target_path\"] = df.file_path.str.replace(\"Source-data\", \"Final-data\")\n",
    "\n",
    "\t# Get target path\n",
    "\ttarget_path = list(df.target_path.unique())[0]\n",
    "\n",
    "\t# When we open the dataframe file, the lists and dictionaries turn into strings - change them back\n",
    "\tfor column in [\"space-after-information\", 'fwd_align_dict', 'bwd_align_dict', 'substituted_words', \"source_indices\"]:\n",
    "\t\tdf[column] = df[column].astype(\"str\")\n",
    "\t\tdf[column] = df[column].apply(lambda x: ast.literal_eval(x))\n",
    "\n",
    "\t# Create lists of information that we need to add to the conllu file\n",
    "\tids_list = df.sentence_id.to_list()\n",
    "\tsource_text = df.text.to_list()\n",
    "\t# initial_translation = df.translation.to_list()\n",
    "\tspace_after_list = df[\"space-after-information\"].to_list()\n",
    "\tfwd_align_list = df['fwd_align_dict'].to_list()\n",
    "\tbwd_align_list = df['bwd_align_dict'].to_list()\n",
    "\tsubstituted_words_list = df['substituted_words'].to_list()\n",
    "\t# tokenized_text_list = df[\"source_indices\"].to_list()\n",
    "\t# If translation is empty, replace it with \"/\"\n",
    "\ttrans_list = df.new_translations.to_list()\n",
    "\tsentence_list = []\n",
    "\n",
    "\tfor i in trans_list:\n",
    "\t\tif len(i) == 0:\n",
    "\t\t\tsentence_list.append(\"/\")\n",
    "\t\telse:\n",
    "\t\t\tsentence_list.append(i)\n",
    "\n",
    "\t# To feed the entire list into the pipeline, we need to create lists of tokens, split by space\n",
    "\tsentence_list = [x.split(\" \") for x in sentence_list]\n",
    "\n",
    "\n",
    "\t# Linguistically process the list\n",
    "\tdoc = nlp(sentence_list)\n",
    "\n",
    "\t# Save the conllu file - rename this for troubleshooting\n",
    "\tCoNLL.write_doc2conll(doc, \"{}/results/{}/temp/{}-troubleshoot\".format(main_path, lang_code, file))\n",
    "\n",
    "\tprint(\"{} processed and saved.\".format(file))\n",
    "\n",
    "\t# Open the CONLL-u file with the CONLL-u parser\n",
    "\n",
    "\tdata = open(\"{}/results/{}/temp/{}-troubleshoot\".format(main_path, lang_code, file), \"r\").read()\n",
    "\n",
    "\tsentences = parse(data)\n",
    "\n",
    "\t# Adding additional information to the conllu\n",
    "\tfor sentence in sentences:\n",
    "\t\t# Get the sentence index\n",
    "\t\tsentence_index = sentences.index(sentence)\n",
    "\n",
    "\t\t# Add metadata\n",
    "\t\tsentence.metadata[\"sent_id\"] = ids_list[sentence_index]\n",
    "\t\tsentence.metadata[\"source\"] = source_text[sentence_index]\n",
    "\t\t# sentence.metadata[\"source_indices\"] = tokenized_text_list[sentence_index]\n",
    "\t\t# sentence.metadata[\"initial_translation\"] = initial_translation[sentence_index]\n",
    "\n",
    "\t\t# Delete the current metadata for text\n",
    "\t\tdel sentence.metadata[\"text\"]\n",
    "\n",
    "\t\tnew_translation_text = \"\"\n",
    "\n",
    "\t\t# Iterate through tokens\n",
    "\t\tfor word in sentence:\n",
    "\t\t\tword_index = sentence.index(word)\n",
    "\t\t\tword_conllu_index = word[\"id\"]\n",
    "\n",
    "\t\t\t# Check whether the word conllu index (word id) is in the substituted_words_list (it is if it was substituted)\n",
    "\t\t\t# If it is, add information on the original translated word - do not do this for Bulgarian, Portuguese and other languages mentioned\n",
    "\t\t\tif lang_code not in [\"BG\", \"PT\", \"IT\", \"AT\", \"GR\", \"HU\", \"NO\", \"TR\", \"NL\", \"SI\", \"HR\", \"RS\", \"LV\", \"UA\", \"ES-GA\", \"PL\", \"ES-CT\", \"FR\", \"BE-nl\", \"BE-fr\", \"EE\", \"ES\", \"FI\", \"ES-PV-es\", \"ES-PV-eu\"]:\n",
    "\t\t\t\tif substituted_words_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\t\tword[\"misc\"][\"Translated\"] = substituted_words_list[sentence_index][word_conllu_index]\n",
    "\t\t\t\n",
    "\t\t\t# Do the same for the forward and backward alignment\n",
    "\t\t\tif fwd_align_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\tword[\"misc\"][\"ForwardAlignment\"] = fwd_align_list[sentence_index][word_conllu_index]\n",
    "\n",
    "\t\t\tif bwd_align_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\tword[\"misc\"][\"BackwardAlignment\"] = bwd_align_list[sentence_index][word_conllu_index]\n",
    "\n",
    "\t\t\t# Remove information on start_char and end_char from the annotation\n",
    "\t\t\tdel word[\"misc\"][\"start_char\"]\n",
    "\t\t\tdel word[\"misc\"][\"end_char\"]\n",
    "\t\t\t\n",
    "\t\t\t# Change the NER tags so that they are the same as in the source\n",
    "\t\t\tcurrent_ner = word[\"misc\"][\"ner\"]\n",
    "\t\t\tdel word[\"misc\"][\"ner\"]\n",
    "\t\t\t\n",
    "\t\t\t# Substitute parts of the tags so that they are the same as in source\n",
    "\t\t\tcurrent_ner = re.sub(\"S-\", \"B-\", current_ner)\n",
    "\t\t\tcurrent_ner = re.sub(\"E-\", \"I-\", current_ner)\n",
    "\n",
    "\t\t\tword[\"misc\"][\"NER\"] = current_ner\n",
    "\n",
    "\t\t\ttry:\n",
    "\t\t\t\t# Get information about the space after based on the index\n",
    "\t\t\t\tcurrent_space_after = space_after_list[sentence_index][word_index]\n",
    "\t\t\texcept:\n",
    "\t\t\t\tprint(\"Error based on current_space after in sentence {}, sentence index: {}, word {}, word index {}.\".format(sentence, sentence_index, word, word_index))\n",
    "\t\t\t\tcurrent_space_after = \"Yes\"\n",
    "\n",
    "\t\t# Create new text from translation, correcting the spaces around words\n",
    "\t\t# based on the SpaceAfter information\n",
    "\t\t\tif current_space_after == \"No\":\n",
    "\t\t\t\tword[\"misc\"][\"SpaceAfter\"] = \"No\"\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\telif current_space_after == \"Last\":\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\telse:\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\t\tnew_translation_text += \" \"\n",
    "\t\t\n",
    "\t\tsentence.metadata[\"text\"] = new_translation_text\n",
    "\t\n",
    "\"\"\"\n",
    "\t# Create a new conllu file with the updated information\n",
    "\t# do not save - for troubleshooting\n",
    "\n",
    "\tos.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
    "\tfinal_file = open(\"{}\".format(target_path), \"w\")\n",
    "\n",
    "\tfor sentence in sentences:\n",
    "\t\tfinal_file.write(sentence.serialize())\n",
    "\t\n",
    "\tfinal_file.close()\n",
    "\n",
    "\tprint(\"Final file {} is saved.\".format(target_path))\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def produce_final_conllu(lang_code, final_dataframe):\n",
    "\timport pandas as pd\n",
    "\timport stanza\n",
    "\timport time\n",
    "\tfrom stanza.pipeline.core import DownloadMethod\n",
    "\t\n",
    "\tdf = pd.read_csv(\"{}\".format(final_dataframe), sep=\"\\t\", index_col=0, na_filter = False)\n",
    "\n",
    "\t# Extract only the problematic file\n",
    "\tdf = df[df[\"file\"] == \"ParlaMint-UA_2023-10-17-m0.conllu\"]\n",
    "\n",
    "\t# Create a list of files\n",
    "\tfiles = list(df.file.unique())\n",
    "\t\n",
    "\tstart_time = time.time()\n",
    "\n",
    "\tprint(\"Processing started.\")\n",
    "\n",
    "\t# Define the pipeline, instruct it to use a specific package: \tCoNLL03\n",
    "\tnlp = stanza.Pipeline(lang='en', processors=\"tokenize,mwt,pos,lemma,ner\", package={\"ner\": [\"conll03\"]}, tokenize_pretokenized=True, download_method=DownloadMethod.REUSE_RESOURCES, use_gpu=True)\n",
    "\n",
    "\tfor file in files:\n",
    "\t\tsen_list = create_conllu(file, lang_code, main_path, final_dataframe, nlp)\n",
    "\t\tcurrent_end_time = round((time.time() - start_time)/60,2)\n",
    "\t\tprint(\"Current running time: {}\".format(current_end_time))\n",
    "\t\n",
    "\tend_time = round((time.time() - start_time)/60,2)\n",
    "\n",
    "\tprint(\"Processing completed. It took {} minutes.\".format(end_time))\n",
    "\n",
    "\treturn sen_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>file</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>text</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>proper_nouns</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>translation-tokenized</th>\n",
       "      <th>space-after-information</th>\n",
       "      <th>fwd_align_dict</th>\n",
       "      <th>bwd_align_dict</th>\n",
       "      <th>alignments</th>\n",
       "      <th>new_translations</th>\n",
       "      <th>substitution_info</th>\n",
       "      <th>substituted_pairs</th>\n",
       "      <th>substituted_words</th>\n",
       "      <th>errors</th>\n",
       "      <th>source_indices</th>\n",
       "      <th>post-processed_translations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004780</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u1.p1.lang1.s1</td>\n",
       "      <td>Слава Україні!</td>\n",
       "      <td>Слава Україні !</td>\n",
       "      <td>{}</td>\n",
       "      <td>2</td>\n",
       "      <td>Glory to Ukraine!</td>\n",
       "      <td>Glory to Ukraine !</td>\n",
       "      <td>['Yes', 'Yes', 'No', 'Last']</td>\n",
       "      <td>{1: '1', 2: '1', 3: '2', 4: '3'}</td>\n",
       "      <td>{1: '1', 3: '2', 4: '3'}</td>\n",
       "      <td>{0: 0, 1: 2, 2: 3}</td>\n",
       "      <td>Glory to Ukraine !</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Слава', 1], ['Україні', 2], ['!', 3]]</td>\n",
       "      <td>Glory to Ukraine !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004781</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u1.p2.lang1.s1</td>\n",
       "      <td>Шановні народні депутати України, відповідно д...</td>\n",
       "      <td>Шановні народні депутати України , відповідно ...</td>\n",
       "      <td>{}</td>\n",
       "      <td>30</td>\n",
       "      <td>Honourable National Deputies of Ukraine, in ac...</td>\n",
       "      <td>Honourable National Deputies of Ukraine , in a...</td>\n",
       "      <td>['Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 5: '4', 6: '5...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 4, 4: 5, 5: 7, 6: 8, 7: ...</td>\n",
       "      <td>Honourable National Deputies of Ukraine , in a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Шановні', 1], ['народні', 2], ['депутати', ...</td>\n",
       "      <td>Honourable National Deputies of Ukraine , in a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004782</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u1.p3.lang1.s1</td>\n",
       "      <td>Шановні колеги, звертаю увагу, що згідно з при...</td>\n",
       "      <td>Шановні колеги , звертаю увагу , що згідно з п...</td>\n",
       "      <td>{}</td>\n",
       "      <td>40</td>\n",
       "      <td>Dear colleagues, I point out that, according t...</td>\n",
       "      <td>Dear colleagues , I point out that , according...</td>\n",
       "      <td>['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'No'...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 5: '4', 6: '5...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 4, 4: 5, 5: 7, 6: 6, 7: ...</td>\n",
       "      <td>Dear colleagues , I point out that , according...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Шановні', 1], ['колеги', 2], [',', 3], ['зв...</td>\n",
       "      <td>Dear colleagues , I point out that , according...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004783</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u1.p3.lang1.s2</td>\n",
       "      <td>Я прошу всіх колег неухильно дотримуватись при...</td>\n",
       "      <td>Я прошу всіх колег неухильно дотримуватись при...</td>\n",
       "      <td>{}</td>\n",
       "      <td>21</td>\n",
       "      <td>I ask all colleagues to abide by the decision ...</td>\n",
       "      <td>I ask all colleagues to abide by the decision ...</td>\n",
       "      <td>['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 6: '5, 6', 10...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 5, 6: 9, 7: ...</td>\n",
       "      <td>I ask all colleagues to abide by the decision ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Я', 1], ['прошу', 2], ['всіх', 3], ['колег'...</td>\n",
       "      <td>I ask all colleagues to abide by the decision ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004784</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u1.p4.lang1.s1</td>\n",
       "      <td>Шановні колеги, у разі повітряної тривоги буде...</td>\n",
       "      <td>Шановні колеги , у разі повітряної тривоги буд...</td>\n",
       "      <td>{}</td>\n",
       "      <td>32</td>\n",
       "      <td>Dear colleagues, in the event of air anxiety, ...</td>\n",
       "      <td>Dear colleagues , in the event of air anxiety ...</td>\n",
       "      <td>['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 7, 6: 8, 7: ...</td>\n",
       "      <td>Dear colleagues , in the event of air anxiety ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Шановні', 1], ['колеги', 2], [',', 3], ['у'...</td>\n",
       "      <td>Dear colleagues , in the event of air anxiety ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006226</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u145.p21.lang1.s1</td>\n",
       "      <td>По фракціях покажіть, будь ласка.</td>\n",
       "      <td>По фракціях покажіть , будь ласка .</td>\n",
       "      <td>{}</td>\n",
       "      <td>5</td>\n",
       "      <td>Show the factions, please.</td>\n",
       "      <td>Show the factions , please .</td>\n",
       "      <td>['Yes', 'Yes', 'No', 'Yes', 'No', 'Last']</td>\n",
       "      <td>{1: '3', 2: '1', 3: '2', 4: '4', 5: '5', 6: '7'}</td>\n",
       "      <td>{1: '1', 3: '2, 3', 4: '4', 5: '5, 6', 6: '7'}</td>\n",
       "      <td>{0: 0, 1: 2, 2: 2, 3: 3, 4: 4, 5: 4, 6: 5}</td>\n",
       "      <td>Show the factions , please .</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['По', 1], ['фракціях', 2], ['покажіть', 3], ...</td>\n",
       "      <td>Show the factions , please .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006227</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u145.p22.lang1.s1</td>\n",
       "      <td>Шановні колеги, розгляд питань порядку денного...</td>\n",
       "      <td>Шановні колеги , розгляд питань порядку денног...</td>\n",
       "      <td>{}</td>\n",
       "      <td>14</td>\n",
       "      <td>Dear colleagues, the consideration of the agen...</td>\n",
       "      <td>Dear colleagues , the consideration of the age...</td>\n",
       "      <td>['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 7: '6...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 5: '4', 7: '5', 8: '6...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 4, 4: 6, 5: 7, 6: 7, 7: ...</td>\n",
       "      <td>Dear colleagues , the consideration of the age...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Шановні', 1], ['колеги', 2], [',', 3], ['ро...</td>\n",
       "      <td>Dear colleagues , the consideration of the age...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006228</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u145.p23.lang1.s1</td>\n",
       "      <td>Зараз я оголошую перерву у нашому пленарному з...</td>\n",
       "      <td>Зараз я оголошую перерву у нашому пленарному з...</td>\n",
       "      <td>{}</td>\n",
       "      <td>20</td>\n",
       "      <td>I now announce a recess in our plenary session...</td>\n",
       "      <td>I now announce a recess in our plenary session...</td>\n",
       "      <td>['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...</td>\n",
       "      <td>{1: '2', 2: '1', 3: '3', 4: '4', 5: '4', 6: '5...</td>\n",
       "      <td>{2: '1', 1: '2', 3: '3', 5: '4', 6: '5', 7: '6...</td>\n",
       "      <td>{0: 1, 1: 0, 2: 2, 3: 4, 4: 5, 5: 6, 6: 7, 7: ...</td>\n",
       "      <td>I now announce a recess in our plenary session...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Зараз', 1], ['я', 2], ['оголошую', 3], ['пе...</td>\n",
       "      <td>I now announce a recess in our plenary session...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006229</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u145.p23.lang1.s2</td>\n",
       "      <td>Нагадую на необхідності непоширення інформації...</td>\n",
       "      <td>Нагадую на необхідності непоширення інформації...</td>\n",
       "      <td>{}</td>\n",
       "      <td>8</td>\n",
       "      <td>I remind you of the need for non-proliferation...</td>\n",
       "      <td>I remind you of the need for non-proliferation...</td>\n",
       "      <td>['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...</td>\n",
       "      <td>{1: '1', 2: '1', 3: '1', 6: '3', 7: '3', 8: '4...</td>\n",
       "      <td>{2: '1', 4: '2', 6: '3', 8: '4', 10: '5', 11: ...</td>\n",
       "      <td>{0: 1, 1: 3, 2: 5, 3: 7, 4: 9, 5: 10, 6: 11, 7...</td>\n",
       "      <td>I remind you of the need for non-proliferation...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Нагадую', 1], ['на', 2], ['необхідності', 3...</td>\n",
       "      <td>I remind you of the need for non-proliferation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006230</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.conllu</td>\n",
       "      <td>ParlaMint-UA_2023-10-17-m0.u145.p24.lang1.s1</td>\n",
       "      <td>Разом до перемоги і Слава Україні!</td>\n",
       "      <td>Разом до перемоги і Слава Україні !</td>\n",
       "      <td>{}</td>\n",
       "      <td>6</td>\n",
       "      <td>Together to victory and the glory of Ukraine!</td>\n",
       "      <td>Together to victory and the glory of Ukraine !</td>\n",
       "      <td>['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...</td>\n",
       "      <td>{0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 7, 6: 8}</td>\n",
       "      <td>Together to victory and the glory of Ukraine !</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>No</td>\n",
       "      <td>[['Разом', 1], ['до', 2], ['перемоги', 3], ['і...</td>\n",
       "      <td>Together to victory and the glory of Ukraine !</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1451 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 file_path  \\\n",
       "2004780  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2004781  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2004782  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2004783  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2004784  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "...                                                    ...   \n",
       "2006226  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2006227  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2006228  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2006229  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "2006230  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "\n",
       "                                      file  \\\n",
       "2004780  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2004781  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2004782  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2004783  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2004784  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "...                                    ...   \n",
       "2006226  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2006227  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2006228  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2006229  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "2006230  ParlaMint-UA_2023-10-17-m0.conllu   \n",
       "\n",
       "                                          sentence_id  \\\n",
       "2004780     ParlaMint-UA_2023-10-17-m0.u1.p1.lang1.s1   \n",
       "2004781     ParlaMint-UA_2023-10-17-m0.u1.p2.lang1.s1   \n",
       "2004782     ParlaMint-UA_2023-10-17-m0.u1.p3.lang1.s1   \n",
       "2004783     ParlaMint-UA_2023-10-17-m0.u1.p3.lang1.s2   \n",
       "2004784     ParlaMint-UA_2023-10-17-m0.u1.p4.lang1.s1   \n",
       "...                                               ...   \n",
       "2006226  ParlaMint-UA_2023-10-17-m0.u145.p21.lang1.s1   \n",
       "2006227  ParlaMint-UA_2023-10-17-m0.u145.p22.lang1.s1   \n",
       "2006228  ParlaMint-UA_2023-10-17-m0.u145.p23.lang1.s1   \n",
       "2006229  ParlaMint-UA_2023-10-17-m0.u145.p23.lang1.s2   \n",
       "2006230  ParlaMint-UA_2023-10-17-m0.u145.p24.lang1.s1   \n",
       "\n",
       "                                                      text  \\\n",
       "2004780                                     Слава Україні!   \n",
       "2004781  Шановні народні депутати України, відповідно д...   \n",
       "2004782  Шановні колеги, звертаю увагу, що згідно з при...   \n",
       "2004783  Я прошу всіх колег неухильно дотримуватись при...   \n",
       "2004784  Шановні колеги, у разі повітряної тривоги буде...   \n",
       "...                                                    ...   \n",
       "2006226                  По фракціях покажіть, будь ласка.   \n",
       "2006227  Шановні колеги, розгляд питань порядку денного...   \n",
       "2006228  Зараз я оголошую перерву у нашому пленарному з...   \n",
       "2006229  Нагадую на необхідності непоширення інформації...   \n",
       "2006230                 Разом до перемоги і Слава Україні!   \n",
       "\n",
       "                                            tokenized_text proper_nouns  \\\n",
       "2004780                                    Слава Україні !           {}   \n",
       "2004781  Шановні народні депутати України , відповідно ...           {}   \n",
       "2004782  Шановні колеги , звертаю увагу , що згідно з п...           {}   \n",
       "2004783  Я прошу всіх колег неухильно дотримуватись при...           {}   \n",
       "2004784  Шановні колеги , у разі повітряної тривоги буд...           {}   \n",
       "...                                                    ...          ...   \n",
       "2006226                По фракціях покажіть , будь ласка .           {}   \n",
       "2006227  Шановні колеги , розгляд питань порядку денног...           {}   \n",
       "2006228  Зараз я оголошую перерву у нашому пленарному з...           {}   \n",
       "2006229  Нагадую на необхідності непоширення інформації...           {}   \n",
       "2006230                Разом до перемоги і Слава Україні !           {}   \n",
       "\n",
       "         length                                        translation  \\\n",
       "2004780       2                                  Glory to Ukraine!   \n",
       "2004781      30  Honourable National Deputies of Ukraine, in ac...   \n",
       "2004782      40  Dear colleagues, I point out that, according t...   \n",
       "2004783      21  I ask all colleagues to abide by the decision ...   \n",
       "2004784      32  Dear colleagues, in the event of air anxiety, ...   \n",
       "...         ...                                                ...   \n",
       "2006226       5                         Show the factions, please.   \n",
       "2006227      14  Dear colleagues, the consideration of the agen...   \n",
       "2006228      20  I now announce a recess in our plenary session...   \n",
       "2006229       8  I remind you of the need for non-proliferation...   \n",
       "2006230       6      Together to victory and the glory of Ukraine!   \n",
       "\n",
       "                                     translation-tokenized  \\\n",
       "2004780                                 Glory to Ukraine !   \n",
       "2004781  Honourable National Deputies of Ukraine , in a...   \n",
       "2004782  Dear colleagues , I point out that , according...   \n",
       "2004783  I ask all colleagues to abide by the decision ...   \n",
       "2004784  Dear colleagues , in the event of air anxiety ...   \n",
       "...                                                    ...   \n",
       "2006226                       Show the factions , please .   \n",
       "2006227  Dear colleagues , the consideration of the age...   \n",
       "2006228  I now announce a recess in our plenary session...   \n",
       "2006229  I remind you of the need for non-proliferation...   \n",
       "2006230     Together to victory and the glory of Ukraine !   \n",
       "\n",
       "                                   space-after-information  \\\n",
       "2004780                       ['Yes', 'Yes', 'No', 'Last']   \n",
       "2004781  ['Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes...   \n",
       "2004782  ['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'No'...   \n",
       "2004783  ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...   \n",
       "2004784  ['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes...   \n",
       "...                                                    ...   \n",
       "2006226          ['Yes', 'Yes', 'No', 'Yes', 'No', 'Last']   \n",
       "2006227  ['Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes...   \n",
       "2006228  ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...   \n",
       "2006229  ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...   \n",
       "2006230  ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Ye...   \n",
       "\n",
       "                                            fwd_align_dict  \\\n",
       "2004780                   {1: '1', 2: '1', 3: '2', 4: '3'}   \n",
       "2004781  {1: '1', 2: '2', 3: '3', 4: '4', 5: '4', 6: '5...   \n",
       "2004782  {1: '1', 2: '2', 3: '3', 4: '4', 5: '4', 6: '5...   \n",
       "2004783  {1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6...   \n",
       "2004784  {1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...   \n",
       "...                                                    ...   \n",
       "2006226   {1: '3', 2: '1', 3: '2', 4: '4', 5: '5', 6: '7'}   \n",
       "2006227  {1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 7: '6...   \n",
       "2006228  {1: '2', 2: '1', 3: '3', 4: '4', 5: '4', 6: '5...   \n",
       "2006229  {1: '1', 2: '1', 3: '1', 6: '3', 7: '3', 8: '4...   \n",
       "2006230  {1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...   \n",
       "\n",
       "                                            bwd_align_dict  \\\n",
       "2004780                           {1: '1', 3: '2', 4: '3'}   \n",
       "2004781  {1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 8: '6...   \n",
       "2004782  {1: '1', 2: '2', 3: '3', 5: '4', 6: '5', 8: '6...   \n",
       "2004783  {1: '1', 2: '2', 3: '3', 4: '4', 6: '5, 6', 10...   \n",
       "2004784  {1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...   \n",
       "...                                                    ...   \n",
       "2006226     {1: '1', 3: '2, 3', 4: '4', 5: '5, 6', 6: '7'}   \n",
       "2006227  {1: '1', 2: '2', 3: '3', 5: '4', 7: '5', 8: '6...   \n",
       "2006228  {2: '1', 1: '2', 3: '3', 5: '4', 6: '5', 7: '6...   \n",
       "2006229  {2: '1', 4: '2', 6: '3', 8: '4', 10: '5', 11: ...   \n",
       "2006230  {1: '1', 2: '2', 3: '3', 4: '4', 6: '5', 8: '6...   \n",
       "\n",
       "                                                alignments  \\\n",
       "2004780                                 {0: 0, 1: 2, 2: 3}   \n",
       "2004781  {0: 0, 1: 1, 2: 2, 3: 4, 4: 5, 5: 7, 6: 8, 7: ...   \n",
       "2004782  {0: 0, 1: 1, 2: 2, 3: 4, 4: 5, 5: 7, 6: 6, 7: ...   \n",
       "2004783  {0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 5, 6: 9, 7: ...   \n",
       "2004784  {0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 7, 6: 8, 7: ...   \n",
       "...                                                    ...   \n",
       "2006226         {0: 0, 1: 2, 2: 2, 3: 3, 4: 4, 5: 4, 6: 5}   \n",
       "2006227  {0: 0, 1: 1, 2: 2, 3: 4, 4: 6, 5: 7, 6: 7, 7: ...   \n",
       "2006228  {0: 1, 1: 0, 2: 2, 3: 4, 4: 5, 5: 6, 6: 7, 7: ...   \n",
       "2006229  {0: 1, 1: 3, 2: 5, 3: 7, 4: 9, 5: 10, 6: 11, 7...   \n",
       "2006230         {0: 0, 1: 1, 2: 2, 3: 3, 4: 5, 5: 7, 6: 8}   \n",
       "\n",
       "                                          new_translations substitution_info  \\\n",
       "2004780                                 Glory to Ukraine !                []   \n",
       "2004781  Honourable National Deputies of Ukraine , in a...                []   \n",
       "2004782  Dear colleagues , I point out that , according...                []   \n",
       "2004783  I ask all colleagues to abide by the decision ...                []   \n",
       "2004784  Dear colleagues , in the event of air anxiety ...                []   \n",
       "...                                                    ...               ...   \n",
       "2006226                       Show the factions , please .                []   \n",
       "2006227  Dear colleagues , the consideration of the age...                []   \n",
       "2006228  I now announce a recess in our plenary session...                []   \n",
       "2006229  I remind you of the need for non-proliferation...                []   \n",
       "2006230     Together to victory and the glory of Ukraine !                []   \n",
       "\n",
       "        substituted_pairs substituted_words errors  \\\n",
       "2004780                 0                {}     No   \n",
       "2004781                 0                {}     No   \n",
       "2004782                 0                {}     No   \n",
       "2004783                 0                {}     No   \n",
       "2004784                 0                {}     No   \n",
       "...                   ...               ...    ...   \n",
       "2006226                 0                {}     No   \n",
       "2006227                 0                {}     No   \n",
       "2006228                 0                {}     No   \n",
       "2006229                 0                {}     No   \n",
       "2006230                 0                {}     No   \n",
       "\n",
       "                                            source_indices  \\\n",
       "2004780           [['Слава', 1], ['Україні', 2], ['!', 3]]   \n",
       "2004781  [['Шановні', 1], ['народні', 2], ['депутати', ...   \n",
       "2004782  [['Шановні', 1], ['колеги', 2], [',', 3], ['зв...   \n",
       "2004783  [['Я', 1], ['прошу', 2], ['всіх', 3], ['колег'...   \n",
       "2004784  [['Шановні', 1], ['колеги', 2], [',', 3], ['у'...   \n",
       "...                                                    ...   \n",
       "2006226  [['По', 1], ['фракціях', 2], ['покажіть', 3], ...   \n",
       "2006227  [['Шановні', 1], ['колеги', 2], [',', 3], ['ро...   \n",
       "2006228  [['Зараз', 1], ['я', 2], ['оголошую', 3], ['пе...   \n",
       "2006229  [['Нагадую', 1], ['на', 2], ['необхідності', 3...   \n",
       "2006230  [['Разом', 1], ['до', 2], ['перемоги', 3], ['і...   \n",
       "\n",
       "                               post-processed_translations  \n",
       "2004780                                 Glory to Ukraine !  \n",
       "2004781  Honourable National Deputies of Ukraine , in a...  \n",
       "2004782  Dear colleagues , I point out that , according...  \n",
       "2004783  I ask all colleagues to abide by the decision ...  \n",
       "2004784  Dear colleagues , in the event of air anxiety ...  \n",
       "...                                                    ...  \n",
       "2006226                       Show the factions , please .  \n",
       "2006227  Dear colleagues , the consideration of the age...  \n",
       "2006228  I now announce a recess in our plenary session...  \n",
       "2006229  I remind you of the need for non-proliferation...  \n",
       "2006230     Together to victory and the glory of Ukraine !  \n",
       "\n",
       "[1451 rows x 20 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"{}\".format(final_dataframe), sep=\"\\t\", index_col=0, na_filter = False)\n",
    "\n",
    "# Extract only the problematic file\n",
    "test_df = df[df[\"file\"] == \"ParlaMint-UA_2023-10-17-m0.conllu\"]\n",
    "\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-21 12:38:36 WARNING: Can not find mwt: default from official model list. Ignoring it.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing started.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-21 12:38:36 INFO: Loading these models for language: en (English):\n",
      "========================\n",
      "| Processor | Package  |\n",
      "------------------------\n",
      "| tokenize  | combined |\n",
      "| pos       | combined |\n",
      "| lemma     | combined |\n",
      "| ner       | conll03  |\n",
      "========================\n",
      "\n",
      "2024-02-21 12:38:36 INFO: Use device: gpu\n",
      "2024-02-21 12:38:36 INFO: Loading: tokenize\n",
      "2024-02-21 12:38:36 INFO: Loading: pos\n",
      "2024-02-21 12:38:37 INFO: Loading: lemma\n",
      "2024-02-21 12:38:37 INFO: Loading: ner\n",
      "2024-02-21 12:38:37 INFO: Done loading processors!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ParlaMint-UA_2023-10-17-m0.conllu processed and saved.\n",
      "Current running time: 1.38\n",
      "Processing completed. It took 1.38 minutes.\n"
     ]
    }
   ],
   "source": [
    "sen_list = produce_final_conllu(lang_code, final_dataframe)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "parlamint_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}