torch_template.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a8e88a2e-d6de-4666-84a2-b9461fdd78cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ONE LIBRARY REQUIRED FOR RUNNING THIS CODE! (aside from torch)\n",
    "# !pip install bi_lstm_crf\n",
    "# Press the 'run all' button to train the model\n",
    "# To train on a custom dataset, replace DATA_PATH with a file containing words with hyphenations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "48319ad0-3943-4460-a4a4-70919aef70b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import time  # For eta\n",
    "\n",
    "# For data management\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# Torch imports\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from bi_lstm_crf import CRF\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e6fff739-5587-4b12-9116-71393387627d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants\n",
    "DATA_PATH = 'sample_dutch_words.txt'  # Filename in directory with words to be hyphenated\n",
    "HYPHENATION_TOKEN = '-'  # Hyphenation character\n",
    "WINDOW_SIZE = 5  # Window size\n",
    "\n",
    "BATCH_SIZE = 64  # Training batch size\n",
    "SPLIT = 0.1  # Train / validation split\n",
    "SUB_SAMPLE = 'FULL'  # Set to custom amount for subsample, leave to full to train whole wordlist\n",
    "\n",
    "REMOVE_APOSTROPHE = 'NO'  # Set to 'YES' to remove apostrophe from input words\n",
    "\n",
    "SEED = 9999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f01e19d3-e072-44f3-b921-0fee2f5b6afe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total length of x/y is: 10000\n",
      "Total alphabet tokens are: 26 + 1\n",
      "Alphabet characters found in source file are: abcdefghijklmnopqrstuvwxyz\n",
      "Max tokens per word is: 30\n"
     ]
    }
   ],
   "source": [
    "# Pre-processing\n",
    "hyphenated_words = pd.read_csv(DATA_PATH, na_values=['xxxp'], keep_default_na=False, header=None)[0]\n",
    "\n",
    "if REMOVE_APOSTROPHE == 'YES':\n",
    "    hyphenated_words.apply(lambda x: x.replace(\"'\", ''))\n",
    "    \n",
    "words = hyphenated_words.apply(lambda x: x.replace(HYPHENATION_TOKEN, '')) \n",
    "ALPHABET = ''.join(sorted(set(''.join(words.to_list()))))\n",
    "MAX_TOKEN_LENGTH = np.max(words.str.len())\n",
    "\n",
    "# Codes hyphenation into solution, e.g. 'num-ber' into '112111'\n",
    "def code_solution(word):\n",
    "    pattern = '.'+HYPHENATION_TOKEN\n",
    "    solut = re.sub(pattern,'2', word)\n",
    "    solut = re.sub('[^2]','1', solut)\n",
    "    return solut\n",
    "\n",
    "# For padding / converting all input to np array\n",
    "def convert(input_string, input_type, direction='code'):\n",
    "    if direction=='code':\n",
    "        if input_type == 'word':\n",
    "            return [(ALPHABET.index(i)+1) for i in input_string] + [0]*(MAX_TOKEN_LENGTH-len(input_string))\n",
    "        elif input_type == 'solution':\n",
    "            return [int(i) for i in input_string] + [0]*(MAX_TOKEN_LENGTH-len(input_string))\n",
    "    if direction=='decode':\n",
    "        if input_type == 'word':\n",
    "            return ''.join([ALPHABET[i-1] for i in input_string if i!=0])\n",
    "        elif input_type == 'solution':\n",
    "            return ''.join([str(i) for i in input_string if i!=0])\n",
    "\n",
    "# Shorthand for converter\n",
    "encode_x = lambda input_string: convert(input_string, input_type='word', direction='code')\n",
    "encode_y = lambda input_string: convert(input_string, input_type='solution', direction='code')\n",
    "decode_x = lambda input_string: convert(input_string, input_type='word', direction='decode')\n",
    "decode_y = lambda input_string: convert(input_string, input_type='solution', direction='decode')\n",
    "\n",
    "# For creating windowed version of input\n",
    "def expand(x_input):\n",
    "    chars = len(ALPHABET)\n",
    "    window_pad = WINDOW_SIZE//2\n",
    "    window = list(range(chars, chars+window_pad*2))\n",
    "    word_padded = np.concatenate([window[0:window_pad], x_input[x_input!=0], window[window_pad:]])\n",
    "    \n",
    "    full_list = []\n",
    "    start_index=0\n",
    "    for i, char in enumerate(x_input):\n",
    "        if char==0:\n",
    "            full_list.append([0]*WINDOW_SIZE)\n",
    "        else:\n",
    "            full_list.append(word_padded[start_index:start_index+WINDOW_SIZE])\n",
    "            start_index+= 1\n",
    "            \n",
    "    full_list = np.array(full_list)  \n",
    "    return full_list  \n",
    "\n",
    "def process_data(hyphenated_words):   \n",
    "    # Process labels\n",
    "    labels = hyphenated_words.apply(code_solution)\n",
    "    labels = labels.apply(encode_y)\n",
    "    labels = np.array(labels.to_list())\n",
    "\n",
    "    # Encode words\n",
    "    words = hyphenated_words.apply(lambda x: x.replace(HYPHENATION_TOKEN, ''))\n",
    "    words = np.array(words.apply(encode_x).to_list())\n",
    "    words_windowed = np.empty((len(words), MAX_TOKEN_LENGTH, WINDOW_SIZE),dtype=np.int32)\n",
    "\n",
    "    for i in range(len(words)):\n",
    "        words_windowed[i] = expand(words[i])\n",
    "\n",
    "    return words_windowed, labels\n",
    "    \n",
    "x, y = process_data(hyphenated_words)\n",
    "\n",
    "# Print\n",
    "print('Total length of x/y is: ' + str(len(hyphenated_words)))\n",
    "print('Total alphabet tokens are: ' + str(len(ALPHABET)) + ' + 1')\n",
    "print('Alphabet characters found in source file are: ' + str(ALPHABET))\n",
    "print('Max tokens per word is: ' + str(MAX_TOKEN_LENGTH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8a9e46ea-6735-492a-b734-cf64b5c8562a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training set length: 9000\n",
      "Total testing set length: 1000\n",
      "Sample train word: rokbanden, 112112111\n",
      "Sample test word: laadpoorten, 11121112111\n"
     ]
    }
   ],
   "source": [
    "if SUB_SAMPLE == 'FULL':\n",
    "    SUB_SAMPLE = len(x)\n",
    "\n",
    "# Define rng, pick random indexes\n",
    "rng = np.random.default_rng(seed=SEED)\n",
    "indexes = rng.choice(hyphenated_words.index,replace=False,size=SUB_SAMPLE)\n",
    "x_train = x[indexes[:int(SUB_SAMPLE*(1-SPLIT))]]\n",
    "y_train = y[indexes[:int(SUB_SAMPLE*(1-SPLIT))]]\n",
    "x_test = x[indexes[int(SUB_SAMPLE*(1-SPLIT)):]]\n",
    "y_test = y[indexes[int(SUB_SAMPLE*(1-SPLIT)):]]\n",
    "\n",
    "# Print\n",
    "train_int = np.random.randint(len(x_train)-1)\n",
    "test_int = np.random.randint(len(x_test)-1)\n",
    "\n",
    "print('Total training set length: ' + str(len(x_train)))\n",
    "print('Total testing set length: ' + str(len(x_test)))\n",
    "print('Sample train word: ' + decode_x(x_train[train_int][:,2]) + ', ' + decode_y(y_train[train_int]))\n",
    "print('Sample test word: ' + decode_x(x_test[test_int][:,2]) + ', ' + decode_y(y_test[test_int]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7ce47382-ec38-4f28-be1c-21ed1d14ad4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pytorch data preperation\n",
    "ALPHABET_SIZE = len(ALPHABET)+WINDOW_SIZE-1\n",
    "\n",
    "# Create dataset, loader\n",
    "train_data = TensorDataset(torch.tensor(x_train, dtype=torch.long), torch.tensor(y_train, dtype=torch.long))\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_data = TensorDataset(torch.tensor(x_test, dtype=torch.long), torch.tensor(y_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "249712b1-585c-4d2e-bc0a-14da17235a4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function for comparing predicted with true performance\n",
    "def callback_word_char_accuracy(y_predict, y_true):\n",
    "    outputs_tensor = torch.nested.nested_tensor(y_predict)\n",
    "    outputs_tensor = torch.nested.to_padded_tensor(outputs_tensor, padding=0,output_size=(len(y_predict),MAX_TOKEN_LENGTH))\n",
    "    outputs_tensor = outputs_tensor.to(device)\n",
    "    \n",
    "    comparison_matrix = torch.eq(outputs_tensor, y_true)\n",
    "    \n",
    "    characters_correct = comparison_matrix.sum().item()\n",
    "    words_correct = (comparison_matrix.all(dim=1)).sum().item()\n",
    "    return characters_correct, words_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35c9da58-00da-436c-b10b-908bf3bb1af6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "OUT_CHANNELS = 40\n",
    "CONV_KERNEL_SIZE = 3\n",
    "POOL_KERNEL_SIZE = 3\n",
    "LSTM_UNITS = 128\n",
    "DROPOUT_EM = 0.3\n",
    "\n",
    "LSTM_IN = ((WINDOW_SIZE+2)-POOL_KERNEL_SIZE+1) * OUT_CHANNELS\n",
    "\n",
    "# Pytorch modeling\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "# 1. Define model\n",
    "class syl_model(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(syl_model, self).__init__()\n",
    "        self.em = nn.Embedding(ALPHABET_SIZE, 128)\n",
    "        self.dropout1 = nn.Dropout(p=DROPOUT_EM)\n",
    "        self.conv1 =  torch.nn.Conv1d(in_channels=128, out_channels=OUT_CHANNELS, kernel_size=CONV_KERNEL_SIZE, stride=1, padding='same')\n",
    "        self.bn1 = nn.BatchNorm1d(OUT_CHANNELS)  # Add batch normalization after conv layer\n",
    "        self.pool = torch.nn.MaxPool1d(kernel_size=POOL_KERNEL_SIZE, stride=1, padding=1)\n",
    "        \n",
    "        self.lstm = nn.LSTM(LSTM_IN, LSTM_UNITS, batch_first=True, bidirectional=True)\n",
    "        self.bn2 = nn.BatchNorm1d(LSTM_UNITS*2)  # Add batch normalization after LSTM\n",
    "        self.crf = CRF(LSTM_UNITS*2, 3)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        batch_size=x.size(0)\n",
    "        mask = x[:,:,2].gt(0)\n",
    "        \n",
    "        x = x.reshape(batch_size*MAX_TOKEN_LENGTH, WINDOW_SIZE)\n",
    "        x = self.em(x)\n",
    "        x = self.dropout1(x)\n",
    "        x = x.permute(0, 2, 1)\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.pool(x)\n",
    "        x = torch.flatten(x, start_dim=1)\n",
    "        x = x.reshape(batch_size, MAX_TOKEN_LENGTH, LSTM_IN)      \n",
    "        x, _ = self.lstm(x)\n",
    "\n",
    "        x = x.permute(0, 2, 1)  # Change shape for batch norm\n",
    "        x = self.bn2(x)  # Apply batch norm after LSTM\n",
    "        x = x.permute(0, 2, 1)  # Change shape back  \n",
    "        \n",
    "        scores, tag_seq = self.crf(x, mask)      \n",
    "        return scores, tag_seq\n",
    "\n",
    "    def loss(self, x, tags):\n",
    "\n",
    "        batch_size=x.size(0)\n",
    "        mask = x[:,:,2].gt(0)\n",
    "        \n",
    "        x = x.reshape(batch_size*MAX_TOKEN_LENGTH, WINDOW_SIZE)\n",
    "        x = self.em(x)\n",
    "        x = self.dropout1(x)\n",
    "        x = x.permute(0, 2, 1)\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn1(x)      \n",
    "        x = F.relu(x)\n",
    "        x = self.pool(x)             \n",
    "        x = torch.flatten(x, start_dim=1)              \n",
    "        x = x.reshape(batch_size, MAX_TOKEN_LENGTH, LSTM_IN)                 \n",
    "        x, _ = self.lstm(x)\n",
    "\n",
    "        x = x.permute(0, 2, 1)  # Change shape for batch norm\n",
    "        x = self.bn2(x)  # Apply batch norm after LSTM\n",
    "        x = x.permute(0, 2, 1)  # Change shape back  \n",
    "        \n",
    "        loss = self.crf.loss(x, tags, mask)\n",
    "        return loss\n",
    "\n",
    "# 2. Create instance, set compiler / loss calculator\n",
    "syl_model_instance = syl_model().to(device)\n",
    "print(syl_model_instance)\n",
    "\n",
    "# 3. Set optimizer\n",
    "optimizer = optim.Adam(syl_model_instance.parameters(),\n",
    "                      lr=3e-4)\n",
    "\n",
    "# 4. Train model\n",
    "N_EPOCHS = 14\n",
    "print('Starting training...')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    # Training\n",
    "    batch = 0\n",
    "    start_time = time.time() \n",
    "\n",
    "    train_loss = 0.0\n",
    "    syl_model_instance.train()  \n",
    "    for inputs, labels in train_loader:\n",
    "\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        batch += 1\n",
    "        optimizer.zero_grad()  # Reset optimizer\n",
    "        loss = syl_model_instance.loss(inputs, labels)  # Forward pass\n",
    "        \n",
    "        loss.backward()  # Backpass\n",
    "        optimizer.step()  # Update\n",
    "        \n",
    "        train_loss += loss.item()\n",
    "        progress = round((((batch + 1) / (len(train_loader)+1))*100)/2)\n",
    "        eta_seconds = (time.time() - start_time) / batch * (len(train_loader) - batch)\n",
    "        eta_minutes = int(eta_seconds // 60)  # Get the number of minutes\n",
    "        eta_remaining_seconds = int(eta_seconds % 60)  # Get the remaining seconds\n",
    "        \n",
    "        # Print progress bar with ETA in minutes:seconds format\n",
    "        print('Epoch ' + str(epoch+1) + ': [' + progress * '=' + (50 - progress) * '.' + \n",
    "              '] ' + str(batch) + '/' + str(len(train_loader) + 1) + \n",
    "              f' | ETA: {eta_minutes}:{eta_remaining_seconds:02d}', end='\\r')\n",
    "        \n",
    "    # Validation\n",
    "    characters_correct = 0\n",
    "    words_correct = 0\n",
    "    \n",
    "    val_loss = 0.0\n",
    "    syl_model_instance.eval()\n",
    "\n",
    "    for inputs, labels in test_loader:\n",
    "        \n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        outputs = syl_model_instance(inputs)\n",
    "        \n",
    "        chars_valid, words_valid = callback_word_char_accuracy(outputs[1], labels)\n",
    "        characters_correct += chars_valid\n",
    "        words_correct += words_valid \n",
    "    \n",
    "    print('E')  # To circumvent the '\\r'\n",
    "    print(\"Character accuracy: {}\\nWord accuracy: {}\".format(characters_correct/(MAX_TOKEN_LENGTH*len(test_data)),\n",
    "                                                              words_correct/len(test_data)))\n",
    "    print('---')\n",
    "\n",
    "    # Save model for epoch\n",
    "    save_filename = 'dutch_seed' + str(SEED) + '_epoch_' + str(epoch+1) + '.pt'\n",
    "    torch.save(syl_model_instance.state_dict(), save_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "979e762e-681b-4f12-965e-ee0c8278bbde",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8951abb7-4472-467d-802f-5a6601386b75",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e88acc-7379-48ec-a045-d656e2d3b0eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e441b10e-e480-432b-8bec-c8d8bb79d803",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input one more more words to syllabificate\n",
    "word_list = 'prima goed leuk en dergelijke andere woorden'\n",
    "target_words = word_list\n",
    "target_words = pd.Series(target_words.split())\n",
    "target_words = np.array(target_words.apply(encode_x).to_list())\n",
    "target_words_windowed = np.empty((len(target_words), MAX_TOKEN_LENGTH, WINDOW_SIZE),dtype=np.int32)\n",
    "for i in range(len(target_words)):\n",
    "    target_words_windowed[i] = expand(target_words[i])\n",
    "target_words_windowed = torch.tensor(target_words_windowed, dtype=torch.long)\n",
    "\n",
    "# Run words through model\n",
    "syl_model_instance.eval()\n",
    "target_outputs = syl_model_instance(target_words_windowed)\n",
    "target_outputs = target_outputs[1]\n",
    "number_words = len(target_outputs)\n",
    "target_outputs = torch.nested.nested_tensor(target_outputs)\n",
    "target_outputs = torch.nested.to_padded_tensor(target_outputs, padding=0,output_size=(number_words,MAX_TOKEN_LENGTH))\n",
    "\n",
    "# Output as string\n",
    "for word in enumerate(word_list.split()):\n",
    "    current_word = []\n",
    "    for char in enumerate(word[1]):\n",
    "        if target_outputs[word[0]][char[0]]==1:\n",
    "            current_word += [char[1]]\n",
    "        else:\n",
    "            current_word += [char[1] + '-']\n",
    "    print(''.join(current_word))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}