THESIS-Indonesian-Sign-Language-Detection-w-MediaPipe-BiGRU_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Hazlan's Thesis**\n",
    "\n",
    "## Judul: Analisis Real-Time Deteksi Gerakan Bahasa Isyarat Di Indonesia Menggunakan MediaPipe dan Bidirectional Gated Recurrent Unit (BiGRU)\n",
    "\n",
    "## Dosen Pembimbing: Budi Santosa S.Si M.Sc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Change Log V4:\n",
    "\n",
    "##### - 3 Kata -> 10 Kata\n",
    "##### - Data Preprocessing changed (Split first, then augment)\n",
    "##### - Modify Detection Interface"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Math and Data Processing\n",
    "import os\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import time\n",
    "import shutil\n",
    "\n",
    "# Sklearn\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import multilabel_confusion_matrix, accuracy_score, classification_report\n",
    "\n",
    "# Tensor & Keras\n",
    "from tensorflow.keras.layers import Dense, GRU, Bidirectional, Dropout\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "# Evaluation\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "# Record and Testing Realtime\n",
    "from scipy import stats\n",
    "import mediapipe as mp\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Keypoints using MP Holistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_holistic = mp.solutions.holistic # Holistic model\n",
    "mp_drawing = mp.solutions.drawing_utils # Drawing utilities\n",
    "\n",
    "# ==\n",
    "\n",
    "def mediapipe_detection(image, model):\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB\n",
    "    image.flags.writeable = False                  # Image is no longer writeable\n",
    "    results = model.process(image)                 # Make prediction\n",
    "    image.flags.writeable = True                   # Image is now writeable \n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR\n",
    "    return image, results\n",
    "\n",
    "# ==\n",
    "\n",
    "def draw_styled_landmarks(image, results):\n",
    "    # Draw left hand connections\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4), mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)) \n",
    "    # Draw right hand connections  \n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4), mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Extract Keypoint Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints(results):\n",
    "    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)\n",
    "    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)\n",
    "    return np.concatenate([lh, rh])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Setup Folders for Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path for exported data, numpy arrays\n",
    "# Utility untuk Pembuatan Folder automatis untuk penyimpaman data koordinat lokasi pose, wajah, dan gerakan tangan -> .npy format\n",
    "# BISINDO\n",
    "# Paths for original and augmented data\n",
    "DATA_PATH = os.path.join('/Users/hzlnqodrey/Developer/SKRIPSI/SIDANG/DATASET/BISINDO_DATASET_SIDANG_26Sep2024')\n",
    "AUGMENTED_DATA_PATH = os.path.join('/Users/hzlnqodrey/Developer/SKRIPSI/SIDANG/DATASET/BISINDO_DATASET_AUGMENTED')\n",
    "\n",
    "# # Remove the existing augmented data dir\n",
    "# if os.path.exists(AUGMENTED_DATA_PATH):\n",
    "#     shutil.rmtree(AUGMENTED_DATA_PATH)\n",
    "\n",
    "# # Delete the directory and all its contents\n",
    "# # Create the new augmented data dir\n",
    "# os.makedirs(AUGMENTED_DATA_PATH)\n",
    "\n",
    "# Total Data (BEFORE AUGMENT) = \n",
    "# 30 x 10 = 300 data original\n",
    "\n",
    "# Total Data (AFTER AUGMENT) = \n",
    "# Training ((27x12) + 27 ) x 10  = 3510 data\n",
    "# Testing 3 x 10 = 30 data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actions that we try to detect\n",
    "actions = np.array(['Hai', 'Aku', 'Cinta', 'Kamu', 'Teman', 'Makan', 'Hari Ini', 'Bantu', 'Terima Kasih', 'Maaf'])\n",
    "\n",
    "# Number of sequences and sequence length\n",
    "no_sequences = 30\n",
    "sequence_length = 30\n",
    "no_sequences_augmented = 12  # Number of augmentations per sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN 1x UNTUK BUAT FOLDER SAJA (JANGAN DI RUN LAGI) UNTUK RECORDING DATA \n",
    "for action in actions: \n",
    "    for sequence in range(no_sequences):\n",
    "        try: \n",
    "            os.makedirs(os.path.join(DATA_PATH, action, str(sequence)))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Collect Keypoint Values for Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN 1x UNTUK BUAT FOLDER SAJA (JANGAN DI RUN LAGI) UNTUK RECORDING DATA \n",
    "cap = cv2.VideoCapture(0)\n",
    "# Set mediapipe model \n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    \n",
    "    # NEW LOOP\n",
    "    # Loop through actions\n",
    "    for action in actions:\n",
    "        # Loop through sequences aka videos\n",
    "        for sequence in range(0, no_sequences):\n",
    "            # Loop through video length aka sequence length\n",
    "            for frame_num in range(sequence_length):\n",
    "                \n",
    "                # Read feed\n",
    "                ret, frame = cap.read()\n",
    "\n",
    "                # Save Extracted Landmarks to .npy file\n",
    "                npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "\n",
    "                # Make detections\n",
    "                image, results = mediapipe_detection(frame, holistic)\n",
    "\n",
    "                # Draw landmarks\n",
    "                draw_styled_landmarks(image, results)\n",
    "                \n",
    "                # NEW Apply wait logic\n",
    "                if frame_num == 0: \n",
    "                    cv2.putText(image, 'STARTING COLLECTION', (120,200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,255, 0), 2, cv2.LINE_AA)\n",
    "                    cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,75), cv2.FONT_HERSHEY_SIMPLEX, 1.7, (0, 0, 255), 2, cv2.LINE_AA)\n",
    "                    # Show to screen\n",
    "                    cv2.imshow('Hazlan BISINDO Record Webcam', image)\n",
    "                    cv2.waitKey(3000)\n",
    "                else: \n",
    "                    cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,75), cv2.FONT_HERSHEY_SIMPLEX, 1.7, (0, 0, 255), 2, cv2.LINE_AA)\n",
    "                    # Show to screen\n",
    "                    cv2.imshow('Hazlan BISINDO Record Webcam', image)\n",
    "                \n",
    "                # NEW Export keypoints\n",
    "                keypoints = extract_keypoints(results)\n",
    "                # npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "                cv2.imwrite(npy_path + '.jpg', frame)\n",
    "                np.save(npy_path, keypoints)\n",
    "\n",
    "                # Break gracefully\n",
    "                if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "                    break\n",
    "                    \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "    cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Split Data [Preprocess] \n",
    "\n",
    "####  - TO GET TEST SET  \n",
    "####  - Partition Folder \n",
    "####  - Augment Folder Prep "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# == Label map for the actions\n",
    "label_map = {label:num for num, label in enumerate(actions)}\n",
    "\n",
    "# == Collect all sequence numbers for each action\n",
    "sequence_numbers = list(range(no_sequences))  # [0, 1, 2, ..., 29]\n",
    "print(f\"Sequence Numbers: {sequence_numbers}\")\n",
    "\n",
    "\n",
    "# == Split the sequence numbers into training and test sets (90% train, 10% test)\n",
    "train_sequences, test_sequences = train_test_split(sequence_numbers, test_size=0.1)\n",
    "print(f\"Train sequence numbers: {train_sequences}\")\n",
    "print(f\"Test sequence numbers: {test_sequences}\")\n",
    "\n",
    "\n",
    "# == Initialize lists to store training and test data\n",
    "X_train, y_train, X_test, y_test = [], [], [], []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## == 6.1 Partition Folder Splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load original sequences into X_train and X_test\n",
    "for action in actions:\n",
    "    # label = label_map[action]  # Get the label for this action\n",
    "    \n",
    "    for sequence in range(no_sequences):  # Loop through each sequence (folder)\n",
    "        window = []  # Store the frames for this sequence\n",
    "        for frame_num in range(sequence_length):\n",
    "            # Load the frame from the sequence folder\n",
    "            res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        \n",
    "        # Partition data based on whether the sequence is in train or test\n",
    "        if sequence in train_sequences:\n",
    "            X_train.append(window)\n",
    "            y_train.append(label_map[action])\n",
    "        elif sequence in test_sequences:\n",
    "            X_test.append(window)\n",
    "            y_test.append(label_map[action])\n",
    "\n",
    "# ==\n",
    "\n",
    "y_train = to_categorical(y_train).astype(int)\n",
    "y_test = to_categorical(y_test).astype(int)\n",
    "\n",
    "# == Convert to numpy arrays\n",
    "X_train = np.array(X_train)\n",
    "X_test = np.array(X_test)\n",
    "y_train = np.array(y_train)\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "# Print shapes to confirm\n",
    "print(f\"X_train shape: {X_train.shape}\")\n",
    "print(f\"X_test shape: {X_test.shape}\")\n",
    "print(f\"y_train shape: {y_train.shape}\")\n",
    "print(f\"y_test shape: {y_test.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# == 6.2 AUGMENTATION FOLDER PREP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Sort the train sequences\n",
    "train_sequence_numbers = sorted(train_sequences)  # Sort the train sequences\n",
    "print(f\"Sorted Train Sequence Numbers: {train_sequence_numbers}\")\n",
    "\n",
    "# Create a mapping for original sequences to new indices\n",
    "sequence_mapping = {}\n",
    "# Map sorted train sequences to [0 ... 26]\n",
    "for idx, seq_num in enumerate(train_sequence_numbers):\n",
    "    sequence_mapping[seq_num] = idx\n",
    "\n",
    "# Step 3: Generate augmented sequence numbers starting from 27\n",
    "print(no_sequences_augmented)\n",
    "print(len(train_sequences))\n",
    "\n",
    "start_augmented_sequence = 27\n",
    "end_augmented_sequence = 350\n",
    "augmented_sequence_numbers = list(range(start_augmented_sequence, end_augmented_sequence + 1))\n",
    "\n",
    "print(end_augmented_sequence)\n",
    "print(len(augmented_sequence_numbers))\n",
    "\n",
    "# Print augmented sequence numbers for debugging\n",
    "print(f\"Augmented sequence numbers: {augmented_sequence_numbers}\")\n",
    "\n",
    "# Combine original train sequences and augmented sequences\n",
    "combined_sequences = sorted(train_sequence_numbers + augmented_sequence_numbers)\n",
    "print(f\"Combined sequence numbers (train [just showing, the new sorted is in sequence_mapping] + augmented): {combined_sequences}\")\n",
    "\n",
    "import shutil\n",
    "\n",
    "# Create folder structure for combined data\n",
    "for action in actions:\n",
    "    # Create folders for training data\n",
    "    for seq_num, new_idx in sequence_mapping.items():\n",
    "        folder_path = os.path.join(AUGMENTED_DATA_PATH, action, str(new_idx))\n",
    "        print(f\"Creating folder for training data: {folder_path}\")\n",
    "        try:\n",
    "            os.makedirs(folder_path)  # Create the folder for the new index\n",
    "        except FileExistsError:\n",
    "            pass  # If the folder exists, move on\n",
    "\n",
    "        # Copy original files to the new folder\n",
    "        for frame_num in range(sequence_length):\n",
    "            original_file_path = os.path.join(DATA_PATH, action, str(seq_num), \"{}.npy\".format(frame_num))\n",
    "            new_file_path = os.path.join(folder_path, \"{}.npy\".format(frame_num))\n",
    "            shutil.copy(original_file_path, new_file_path)  # Copy the .npy file\n",
    "            # print(f\"Copied {original_file_path} to {new_file_path}\")  # Print progress\n",
    "\n",
    "    # Create folders for augmented data\n",
    "    for aug_seq_num in augmented_sequence_numbers:\n",
    "        folder_path = os.path.join(AUGMENTED_DATA_PATH, action, str(aug_seq_num))\n",
    "        print(f\"Creating folder for augmented data: {folder_path}\")\n",
    "        try:\n",
    "            os.makedirs(folder_path)  # Create the folder for the augmented index\n",
    "        except FileExistsError:\n",
    "            pass  # If the folder exists, move on\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Augment Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augmentation_rotation(image, angle):\n",
    "    # Rotate the image\n",
    "    array = [15,30,345,330]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        rows, cols, _ = image.shape\n",
    "        M = cv2.getRotationMatrix2D((cols / 2, rows / 2), element, 1)\n",
    "        augmented_image = cv2.warpAffine(image, M, (cols, rows))\n",
    "        image_list.append(augmented_image)\n",
    "    # Show to screen\n",
    "    return image_list\n",
    "\n",
    "def zoomin(image, zoom_factor):\n",
    "    height, width = image.shape[:2]\n",
    "    new_width = int(width * zoom_factor)\n",
    "    new_height = int(height * zoom_factor)\n",
    "    left = int((width - new_width))\n",
    "    top = int((height - new_height))\n",
    "    right = int((width + new_width))\n",
    "    bottom = int((height + new_height))\n",
    "    cropped_image = image[top:bottom, left:right]\n",
    "    zoom_in_width = int(image.shape[1])\n",
    "    zoom_in_height = int(image.shape[0])\n",
    "    zoom_in_image = cv2.resize(cropped_image, (zoom_in_width, zoom_in_height))\n",
    "    return zoom_in_image\n",
    "\n",
    "def augmentation_zoomin(image):\n",
    "    array = [0.8,0.7]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        augmented_image = zoomin(image, element)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list\n",
    "\n",
    "\n",
    "def zoomout(image, zoom_factor):\n",
    "    height, width = image.shape[:2]\n",
    "    new_width = int(width * zoom_factor)\n",
    "    new_height = int(height * zoom_factor)\n",
    "\n",
    "    # Compute the aspect ratio difference\n",
    "    width_ratio = new_width / width\n",
    "    height_ratio = new_height / height\n",
    "    aspect_ratio = min(width_ratio, height_ratio)\n",
    "\n",
    "    # Compute the new dimensions\n",
    "    new_width = int(width * aspect_ratio)\n",
    "    new_height = int(height * aspect_ratio)\n",
    "\n",
    "    # Ensure that the new dimensions are not larger than the original image dimensions\n",
    "    new_width = min(new_width, width)\n",
    "    new_height = min(new_height, height)\n",
    "\n",
    "    # Compute the black border dimensions\n",
    "    border_width = width - new_width\n",
    "    border_height = height - new_height\n",
    "\n",
    "    # Create a black border around the zoomed-out image\n",
    "    border_color = (0, 0, 0)  # Black color\n",
    "    bordered_image = cv2.copyMakeBorder(image, border_height, border_height, border_width, border_width,\n",
    "                                        cv2.BORDER_CONSTANT, value=border_color)\n",
    "    zoomed_out_image = cv2.resize(bordered_image, (width, height))\n",
    "    return zoomed_out_image\n",
    "\n",
    "def augmentation_zoomout(image):\n",
    "    array = [0.8,0.7]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        augmented_image = zoomout(image, element)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list\n",
    "\n",
    "def augmentation_meta(image):\n",
    "    # Adjust brightness and contrast of the image\n",
    "    array = np.arange(0.5, 2.5, 0.5)\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        brightness = element  # Brightness factor (0.0 to 1.0)\n",
    "        contrast = element + 1  # Contrast factor (>1.0 for higher contrast)\n",
    "        augmented_image = cv2.convertScaleAbs(image, alpha=contrast, beta=brightness)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Augment Data Process and Save to AUGMENTED_FOLDER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to augment and save augmented keypoints\n",
    "def augmented_background(frame_aug, file_num, frame_number, action):\n",
    "    seq_num = file_num\n",
    "    augmented_images = (\n",
    "        augmentation_rotation(frame_aug, 45) +\n",
    "        augmentation_zoomin(frame_aug) +\n",
    "        augmentation_zoomout(frame_aug) +\n",
    "        augmentation_meta(frame_aug)\n",
    "    )\n",
    "    \n",
    "    for index, aug_image in enumerate(augmented_images):\n",
    "        image, results = mediapipe_detection(aug_image, holistic)\n",
    "        keypoints = extract_keypoints(results)\n",
    "        npy_path = os.path.join(AUGMENTED_DATA_PATH, action, str(seq_num), str(frame_number))\n",
    "        if not os.path.exists(os.path.join(AUGMENTED_DATA_PATH, action, str(seq_num))):\n",
    "            os.makedirs(os.path.join(AUGMENTED_DATA_PATH, action, str(seq_num)))\n",
    "        print(npy_path)\n",
    "        np.save(npy_path, keypoints)\n",
    "        seq_num += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    \n",
    "    # Loop through actions\n",
    "    for action in actions:\n",
    "        seq_num = 27  # Starting sequence number for augmented sequences\n",
    "        # Loop through sequences\n",
    "        for train_index, sequence in enumerate(train_sequences):\n",
    "            # Loop through sequence length\n",
    "            for frame_num in range(sequence_length):\n",
    "                # [READ - IMPORTANT - SIDANG - v]\n",
    "                # Construct the path for the original .jpg image corresponding to the X_train sequence\n",
    "                npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "                frame = cv2.imread(f'{npy_path}.jpg')  # Load the image from the path\n",
    "\n",
    "                if frame is not None:\n",
    "                    # Augment and save the augmented keypoints\n",
    "                    augmented_background(frame, seq_num, frame_num, action)\n",
    "            \n",
    "            # Increment sequence number to avoid overlap\n",
    "            seq_num += 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. Combining Train Sequences + Augmented Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {label:num for num, label in enumerate(actions)}\n",
    "\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "sequences, labels = [], []\n",
    "for action in actions:\n",
    "    for sequence in range(end_augmented_sequence+1):\n",
    "        window = []\n",
    "        for frame_num in range(sequence_length):\n",
    "            res = np.load(os.path.join(AUGMENTED_DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])\n",
    "\n",
    "# ==\n",
    "X_TRAIN_FINAL = np.array(sequences)\n",
    "y_train_final = to_categorical(labels).astype(int)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. Build and Train BIGRU Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Bidirectional(GRU(64, return_sequences=True, activation='relu', input_shape=(30,126))))\n",
    "model.add(Bidirectional(GRU(128, return_sequences=False, activation='relu')))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dense(actions.shape[0], activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = os.path.join('Logs')\n",
    "tb_callback = TensorBoard(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the remaining data to train and validation\n",
    "x_train, x_val, y_train, y_val = train_test_split(X_TRAIN_FINAL, y_train_final, test_size=0.1, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(x_train, y_train, epochs=120, callbacks=[tb_callback], validation_data=(x_val, y_val), batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analisis dan Evaluasi Hasil (Evaluation and Result's Analysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot training & validation loss values\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation accuracy values\n",
    "plt.plot(history.history['categorical_accuracy'])\n",
    "plt.plot(history.history['val_categorical_accuracy'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11. Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actions[np.argmax(y_test[0])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12. Save Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('BISINDO_30Sep2024_v6.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ================================\n",
    "## CHECK TRAIN LOGS with TENSORBOARD\n",
    "\n",
    "1. Open a CMD\n",
    "2. CD to the Logs train folder (in my case: \"C:\\.\"YOUR NAME\"\\SKRIPSI-Sign Language\\SCRIPT\\2_TRAINING\\Logs\\train\")\n",
    "3. And Type below\n",
    "\n",
    "tensorboard --logdir=."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "==="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 13. Evaluation - Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ytrue = np.argmax(y_test, axis=1).tolist()\n",
    "yhat = np.argmax(yhat, axis=1).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multilabel_confusion_matrix(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_score(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define class labels\n",
    "class_labels = ['Hai', 'Aku', 'Cinta', 'Kamu', 'Teman', 'Makan', 'Hari Ini', 'Bantu', 'Terima Kasih', 'Maaf']\n",
    "\n",
    "# ytrue and yhat are the predicted and the actual labels\n",
    "conf_matrix = confusion_matrix(ytrue, yhat)\n",
    "\n",
    "accuracy = accuracy_score(ytrue, yhat)\n",
    "precision = precision_score(ytrue, yhat, average='weighted')\n",
    "recall = recall_score(ytrue, yhat, average='weighted')\n",
    "f1 = f1_score(ytrue, yhat, average='weighted')\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.set(font_scale=1)\n",
    "sns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False,\n",
    "            xticklabels=class_labels, yticklabels=class_labels)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('Actual')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.show()\n",
    "\n",
    "print(\"\\nAccuracy:\", accuracy)\n",
    "print(\"Precision:\", precision)\n",
    "print(\"Recall:\", recall)\n",
    "print(\"F1-Score:\", f1)\n",
    "\n",
    "# Calculate classification report\n",
    "report = classification_report(ytrue, yhat, target_names=class_labels)\n",
    "print(\"\\nClassification Report:\\n\", report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 14. Real-Time Test Program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('BISINDO.h5')\n",
    "# model.load_weights('SIBI.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [(245,117,16), (117,245,16), (16,117,245), (120,35,140), (200,13,160), (15,20,200), (103, 156, 151), (96, 145, 98), (140,100, 10), (130, 88, 176)]\n",
    "def prob_viz(res, actions, input_frame, colors):\n",
    "    output_frame = input_frame.copy()\n",
    "    for num, prob in enumerate(res):\n",
    "        # MAC OS FRAME RECTANGLE\n",
    "        \n",
    "        # cv2.rectangle(image, start_point, end_point, color, thickness)\n",
    "        # cv2.putText(image, text, org (coordinates), font, fontScale, color[, thickness[, lineType[, bottomLeftOrigin]]])\n",
    "        \n",
    "        # Pembesaran Frame Rectangle = WORKED but masih KECIL\n",
    "        cv2.rectangle(output_frame, (0, 120+num*100), (int(prob*450), 200+num*100), colors[num], -1)\n",
    "        cv2.putText(output_frame, actions[num], (0, 160+num*100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)\n",
    "\n",
    "        # Kasih Tambahan Angka Probabilitas \n",
    "        cv2.rectangle(output_frame, (500, 130+num*100), (700, 200+num*100), (255, 255, 255), 2)\n",
    "        cv2.putText(output_frame, ' ' + str(round(prob*100, 1)) + '%', (500, 160+num*100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)\n",
    "        \n",
    "    return output_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. New detection variables\n",
    "sequence = []\n",
    "sentence = []\n",
    "predictions = []\n",
    "threshold = 0.9\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "# Set mediapipe model \n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "\n",
    "        # Read feed\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        # Make detections\n",
    "        image, results = mediapipe_detection(frame, holistic)\n",
    "        # print(results)\n",
    "        \n",
    "        # Draw landmarks\n",
    "        draw_styled_landmarks(image, results)\n",
    "        \n",
    "        # 2. Prediction logic\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.append(keypoints)\n",
    "        sequence = sequence[-30:]\n",
    "        \n",
    "        if len(sequence) == 30:\n",
    "            res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "            # print(actions[np.argmax(res)])\n",
    "            predictions.append(np.argmax(res))\n",
    "            \n",
    "            \n",
    "        #3. Viz logic\n",
    "            # # Filter out the noise\n",
    "            if np.argmax(np.bincount(predictions[-10:]))==np.argmax(res):\n",
    "            # if np.unique(predictions[-10:])[0]==np.argmax(res): \n",
    "                if res[np.argmax(res)] > threshold: \n",
    "                    \n",
    "                    if len(sentence) > 0: \n",
    "                        if actions[np.argmax(res)] != sentence[-1]:\n",
    "                            sentence.append(actions[np.argmax(res)])\n",
    "                    else:\n",
    "                        sentence.append(actions[np.argmax(res)])\n",
    "\n",
    "            if len(sentence) > 5: \n",
    "                sentence = sentence[-5:]\n",
    "\n",
    "            # Viz probabilities\n",
    "            image = prob_viz(res, actions, image, colors)\n",
    "            \n",
    "        cv2.rectangle(image, (0,0), (2000, 150), (245, 117, 16), -1)\n",
    "        cv2.putText(image, ' '.join(sentence), (30,100), \n",
    "                       cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0), 2, cv2.LINE_AA)\n",
    "        \n",
    "        # Show to screen\n",
    "        cv2.imshow('Hazlan BISINDO REAL-TIME TEST | WEBCAM', image)\n",
    "\n",
    "        # Break gracefully\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "    cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15. **[Bonus]** Step Perhitungan Arsitektur BiGRU (Simulation Testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== First BiGRU Layer - Forward Cell ===\n",
      "Update Gate (u_f1): 0.6106392741203308\n",
      "Reset Gate (r_f1): 0.6106392741203308\n",
      "Candidate Hidden State (h_tilde_f1): 0.3893756568431854\n",
      "New Hidden State (h_t_f1_new): 0.15160758793354034\n",
      "\n",
      "=== First BiGRU Layer - Backward Cell ===\n",
      "Update Gate (u_b1): 0.6106392741203308\n",
      "Reset Gate (r_b1): 0.6106392741203308\n",
      "Candidate Hidden State (h_tilde_b1): 0.3893756568431854\n",
      "New Hidden State (h_t_b1_new): 0.15160758793354034\n",
      "\n",
      "1st First Cell ReLU: 0.022984860464930534\n",
      "\n",
      "=== Second BiGRU Layer - Forward Cell ===\n",
      "Update Gate (u_f2): 0.6117317080497742\n",
      "Reset Gate (r_f2): 0.6117317080497742\n",
      "Candidate Hidden State (h_tilde_f2): 0.3918512761592865\n",
      "New Hidden State (h_t_f2_new): 0.16620399057865143\n",
      "\n",
      "=== Second BiGRU Layer - Backward Cell ===\n",
      "Update Gate (u_b2): 0.6117317080497742\n",
      "Reset Gate (r_b2): 0.6117317080497742\n",
      "Candidate Hidden State (h_tilde_b2): 0.3918512761592865\n",
      "New Hidden State (h_t_b2_new): 0.16620399057865143\n",
      "\n",
      " 2nd Second Cell ReLU: 0.027623767033219337\n",
      "\n",
      "=== First Dense Layer ===\n",
      "Linear Output (y_linear): 0.2193366438150406\n",
      "\n",
      "=== Second Dense Layer (Output) ===\n",
      "2nd Linear Output (y_linear_2): 0.35353565216064453\n",
      "Softmax Output (y_pred): 1.0\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# Input: Single keypoint value 0.1\n",
    "X = torch.tensor([[0.5]])\n",
    "\n",
    "# Define the GRU network parameters:\n",
    "hidden_size = 1  # GRU layer with only one unit\n",
    "\n",
    "### 2. First BiGRU Layer\n",
    "# Shared weights for update, reset, and new state gates (for simplicity)\n",
    "W_u = torch.tensor([[0.5]])\n",
    "W_r = torch.tensor([[0.5]])\n",
    "W = torch.tensor([[0.5]])  # W for candidate hidden state\n",
    "Q_u = torch.tensor([[0.2]])\n",
    "Q_r = torch.tensor([[0.2]])\n",
    "Q = torch.tensor([[0.2]])  # Q for candidate hidden state\n",
    "b_u = torch.tensor([0.1])\n",
    "b_r = torch.tensor([0.1])\n",
    "b = torch.tensor([0.1])    # b for candidate hidden state\n",
    "\n",
    "# Initialize the hidden state for the forward and backward cells of the first BiGRU layer\n",
    "h_t_f1 = torch.zeros(1)  # Forward cell\n",
    "h_t_b1 = torch.zeros(1)  # Backward cell\n",
    "\n",
    "# GRU cell calculation for the first BiGRU layer - forward cell\n",
    "u_f1 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_f1, Q_u) + b_u)\n",
    "r_f1 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_f1, Q_r) + b_r)\n",
    "h_tilde_f1 = torch.tanh(torch.matmul(X, W) + b + r_f1 * (torch.matmul(h_t_f1, Q) + b))\n",
    "h_t_f1_new = u_f1 * h_t_f1 + (1 - u_f1) * h_tilde_f1\n",
    "\n",
    "# GRU cell calculation for the first BiGRU layer - backward cell\n",
    "u_b1 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_b1, Q_u) + b_u)\n",
    "r_b1 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_b1, Q_r) + b_r)\n",
    "h_tilde_b1 = torch.tanh(torch.matmul(X, W) + b + r_b1 * (torch.matmul(h_t_b1, Q) + b))\n",
    "h_t_b1_new = u_b1 * h_t_b1 + (1 - u_b1) * h_tilde_b1\n",
    "\n",
    "# Combine the forward and backward hidden states using element-wise multiplication\n",
    "h_t_1 = h_t_f1_new * h_t_b1_new\n",
    "\n",
    "# Apply ReLU activation after combining\n",
    "h_t_1_relu = F.relu(h_t_1)\n",
    "\n",
    "### 3. Second BiGRU Layer\n",
    "# Initialize the hidden state for the forward and backward cells of the second BiGRU layer\n",
    "h_t_f2 = torch.zeros(1)  # Forward cell\n",
    "h_t_b2 = torch.zeros(1)  # Backward cell\n",
    "\n",
    "# GRU cell calculation for the second BiGRU layer - forward cell\n",
    "u_f2 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_1_relu, Q_u) + b_u)\n",
    "r_f2 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_1_relu, Q_r) + b_r)\n",
    "h_tilde_f2 = torch.tanh(torch.matmul(X, W) + b + r_f2 * (torch.matmul(h_t_1_relu, Q) + b))\n",
    "h_t_f2_new = u_f2 * h_t_1_relu + (1 - u_f2) * h_tilde_f2\n",
    "\n",
    "# GRU cell calculation for the second BiGRU layer - backward cell\n",
    "u_b2 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_1_relu, Q_u) + b_u)\n",
    "r_b2 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_1_relu, Q_r) + b_r)\n",
    "h_tilde_b2 = torch.tanh(torch.matmul(X, W) + b + r_b2 * (torch.matmul(h_t_1_relu, Q) + b))\n",
    "h_t_b2_new = u_b2 * h_t_1_relu + (1 - u_b2) * h_tilde_b2\n",
    "\n",
    "# Combine the forward and backward hidden states using element-wise multiplication\n",
    "h_t_2 = h_t_f2_new * h_t_b2_new\n",
    "\n",
    "# Apply ReLU activation after combining\n",
    "h_t_2_relu = F.relu(h_t_2)\n",
    "\n",
    "### 4. Dense Layer\n",
    "# Dense layer weights and bias\n",
    "W_dense = torch.tensor([[0.7]])\n",
    "b_dense = torch.tensor([0.2])\n",
    "\n",
    "# Dense layer output\n",
    "y_linear = torch.matmul(h_t_2_relu, W_dense.T) + b_dense\n",
    "\n",
    "# Apply ReLU activation after the dense layer\n",
    "h_dense_relu = F.relu(y_linear)\n",
    "\n",
    "### 5. Output Layer\n",
    "# Softmax activation for output layer\n",
    "y_linear_2 = torch.matmul(h_dense_relu, W_dense.T) + b_dense\n",
    "\n",
    "y_pred = F.softmax(y_linear_2, dim=0)\n",
    "\n",
    "# Print detailed calculations\n",
    "print(\"\\n=== First BiGRU Layer - Forward Cell ===\")\n",
    "print(f\"Update Gate (u_f1): {u_f1.item()}\")\n",
    "print(f\"Reset Gate (r_f1): {r_f1.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_f1): {h_tilde_f1.item()}\")\n",
    "print(f\"New Hidden State (h_t_f1_new): {h_t_f1_new.item()}\")\n",
    "\n",
    "print(\"\\n=== First BiGRU Layer - Backward Cell ===\")\n",
    "print(f\"Update Gate (u_b1): {u_b1.item()}\")\n",
    "print(f\"Reset Gate (r_b1): {r_b1.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_b1): {h_tilde_b1.item()}\")\n",
    "print(f\"New Hidden State (h_t_b1_new): {h_t_b1_new.item()}\")\n",
    "\n",
    "print(f\"\\n1st First Cell ReLU: {h_t_1_relu.item()}\")\n",
    "\n",
    "print(\"\\n=== Second BiGRU Layer - Forward Cell ===\")\n",
    "print(f\"Update Gate (u_f2): {u_f2.item()}\")\n",
    "print(f\"Reset Gate (r_f2): {r_f2.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_f2): {h_tilde_f2.item()}\")\n",
    "print(f\"New Hidden State (h_t_f2_new): {h_t_f2_new.item()}\")\n",
    "\n",
    "print(\"\\n=== Second BiGRU Layer - Backward Cell ===\")\n",
    "print(f\"Update Gate (u_b2): {u_b2.item()}\")\n",
    "print(f\"Reset Gate (r_b2): {r_b2.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_b2): {h_tilde_b2.item()}\")\n",
    "print(f\"New Hidden State (h_t_b2_new): {h_t_b2_new.item()}\")\n",
    "\n",
    "print(f\"\\n 2nd Second Cell ReLU: {h_t_2_relu.item()}\")\n",
    "\n",
    "print(\"\\n=== First Dense Layer ===\")\n",
    "print(f\"Linear Output (y_linear): {y_linear.item()}\")\n",
    "\n",
    "print(\"\\n=== Second Dense Layer (Output) ===\")\n",
    "print(f\"2nd Linear Output (y_linear_2): {y_linear_2.item()}\")\n",
    "print(f\"Softmax Output (y_pred): {y_pred.item()}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}