Demo Creator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "import pandas as pd\n",
    "import sklearn as sk\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import copy\n",
    "import numpy as np\n",
    "import dlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n",
    "if len(physical_devices) > 0:\n",
    "   tf.config.experimental.set_memory_growth(physical_devices[0], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_enged  = keras.models.load_model('C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/Models/engagement86.h5')\n",
    "model_bored  = keras.models.load_model('C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/Models/boredom.h5')\n",
    "model_conf   = keras.models.load_model('C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/confusion.h5')\n",
    "model_frus   = keras.models.load_model('C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/Frustration.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_horizontally(images,padding=0):\n",
    "    max_height = 0  # find the max height of all the images\n",
    "    total_width = 0  # the total width of the images (horizontal stacking)\n",
    "    for img in images:\n",
    "        # open all images and find their sizes\n",
    "        image_height = img.shape[0]\n",
    "        image_width = img.shape[1]\n",
    "        if image_height > max_height:\n",
    "            max_height = image_height\n",
    "        # add all the images widths\n",
    "        total_width += image_width\n",
    "    # create a new array with a size large enough to contain all the images\n",
    "    # also add padding size for all the images except the last one\n",
    "    final_image = np.ones((max_height, (len(images)-1)*padding+ total_width, 3), dtype=np.uint8)\n",
    "    current_x = 0  # keep track of where your current image was last placed in the x coordinate\n",
    "    for image in images:\n",
    "        # add an image to the final array and increment the x coordinate\n",
    "        height = image.shape[0]\n",
    "        width = image.shape[1]\n",
    "        final_image[:height,current_x :width+current_x, :] = image\n",
    "        #add the padding between the images\n",
    "        current_x += width+padding\n",
    "    return final_image"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "image = cv2.imread(\"C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/Data/Test/500044/5000441001/50004410011.jpg\")\n",
    "pre = [0.54,0.68]\n",
    "mat = addBar2Frame(pre)\n",
    "mat = cv2.resize(mat,(image.shape[1],image.shape[0]))\n",
    "print(image.shape)\n",
    "frame = np.concatenate((mat, image), axis=1)\n",
    "cv2.imshow('Example - Show image in window',frame)\n",
    " \n",
    "cv2.waitKey(0) # waits until a key is pressed\n",
    "cv2.destroyAllWindows() # destroys the window showing image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addBar2Frame(predictions, labels):\n",
    "    plt.bar(labels, predictions)\n",
    "    plt.xticks(rotation=16)\n",
    "    plt.style.use('dark_background')\n",
    "#     plt.yticks([],[])\n",
    "    plt.ylim([0,1])\n",
    "    plt.autoscale(enable=False, axis='y')\n",
    "    plt.savefig('temp.png')\n",
    "    mat = cv2.imread('temp.png')\n",
    "    plt.clf()\n",
    "    return mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cap = cv2.VideoCapture(1) # Check if the webcam is opened correctLy \n",
    "if not cap.isOpened():\n",
    "    cap = cv2.VideoCapture('C:/Users/Moh.Massoud/ML/Engagment level/DAiSEE/Demos/demo(hafez).mp4') \n",
    "if not cap.isOpened():\n",
    "    raise IOError(\"Cannot open webcam\")\n",
    "    \n",
    "    \n",
    "frame_width = 1152\n",
    "frame_height = 576\n",
    "\n",
    "size = (frame_width, frame_height)\n",
    "result = cv2.VideoWriter('demo(hafez).avi', \n",
    "                         cv2.VideoWriter_fourcc(*'MJPG'),\n",
    "                         25, size)\n",
    "\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "\n",
    "frame_count = 0\n",
    "state = 0\n",
    "printState = -1\n",
    "count = 0\n",
    "while True:\n",
    "    _, frame = cap.read()\n",
    "    count += 1\n",
    "    if _:\n",
    "        frame_count += 1\n",
    "        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "        faces = detector(gray)\n",
    "        #predictor = dlib.shape_predictor(\"shape_predictor_68_face_landmarks.dat\")\n",
    "\n",
    "        engage=0\n",
    "        roi=[]\n",
    "        for face in faces:\n",
    "            x1 = face.left()\n",
    "            y1 = face.top()\n",
    "            x2 = face.right()\n",
    "            y2 = face.bottom()\n",
    "            # width = x2 - x1\n",
    "            # height = y2 - y1\n",
    "            cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)\n",
    "            if len(frame[y1:y2, x1:x2]) <= 0:\n",
    "                continue\n",
    "            roi.append(cv2.resize(cv2.cvtColor(frame[y1:y2, x1:x2], cv2.COLOR_BGR2GRAY), (48,48)))\n",
    "            predictions_engage = []\n",
    "            predictions_board = []\n",
    "            prediction_frus = []\n",
    "            prediction_conf = []\n",
    "            if(len(roi)>0):\n",
    "                test_images = np.expand_dims(roi, axis=3)\n",
    "                predictions_engage = model_enged.predict(test_images)\n",
    "                predictions_board = model_bored.predict(test_images)\n",
    "                predictions_frus = model_frus.predict_on_batch(test_images)\n",
    "                predictions_conf = model_conf.predict_on_batch(test_images)\n",
    "               \n",
    "            predict = 0        \n",
    "            for i in range(len(predictions_engage)):\n",
    "                predict=np.argmax(predictions_engage[i])\n",
    "            state += predict\n",
    "\n",
    "            if frame_count > 25:\n",
    "                printState = round(state/25,1)\n",
    "                state = 0\n",
    "                frame_count = 0\n",
    "\n",
    "            if printState != -1 and len(roi):    \n",
    "                cv2.putText(frame, f\"{'Engaged' if printState else 'Not Engaged'}\", (x1, y1), font, 0.5, (0, 0, 255), 2, cv2.LINE_4)\n",
    "            predictions = []\n",
    "            predictions.extend(predictions_engage[0])\n",
    "            predictions.extend(predictions_conf[0])\n",
    "            predictions.extend(predictions_board[0])\n",
    "            \n",
    "            predictions.extend(predictions_frus[0])\n",
    "            \n",
    "            predictions_labels = ['not engaged', 'engaged', 'not confused', 'confused', 'not board', 'board', 'not frustrated', 'frustrated']\n",
    "#             print(predictions, len(predictions))\n",
    "#             print(predictions_labels, len(predictions_labels))\n",
    "            mat = addBar2Frame(predictions, predictions_labels)\n",
    "            \n",
    "            frame = combine_horizontally([frame,mat])\n",
    "#             print(frame.shape)\n",
    "            cv2.imshow(\"Drowsiness detection\", frame)\n",
    "            result.write(frame)\n",
    "    else:\n",
    "        cap. release() \n",
    "        cv2.destroyAllWindows() \n",
    "        break\n",
    "    if cv2.waitKey(2) & 0xFF == ord('q'): \n",
    "        cap. release() \n",
    "        cv2.destroyAllWindows() \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}