alr_live_detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import numpy as np\r\n",
    "import os\r\n",
    "import imutils\r\n",
    "import dlib\r\n",
    "import cv2\r\n",
    "import matplotlib.pyplot as plt\r\n",
    "\r\n",
    "import imageio\r\n",
    "from imutils import face_utils"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "detector = dlib.get_frontal_face_detector()\r\n",
    "predictor = dlib.shape_predictor(\"shape_predictor_68_face_landmarks.dat\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "from tensorflow import keras\r\n",
    "model = keras.models.load_model('complete_saved_model_30e/')\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "recording=False\r\n",
    "text = \"---\"\r\n",
    "frames = []\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "words = ['Begin', 'Choose', 'Connection', 'Navigation', 'Next', 'Previous', 'Start', 'Stop', 'Hello', 'Web']\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "import skimage\r\n",
    "from skimage.transform import resize"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "get_ipython().run_line_magic('pwd', '')"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'f:\\\\MIRACLE-V1\\\\ALR'"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "import logging, sys\r\n",
    "import numpy as np"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "\"\"\"image = cv2.VideoCapture(0)\r\n",
    "while True:\r\n",
    " \r\n",
    "    frame = image.read()\r\n",
    "    if frame is not None:  # add this line\r\n",
    "       #print(frame)\r\n",
    "      (H, W) = frame.shape[:2]\"\"\""
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'image = cv2.VideoCapture(0)\\nwhile True:\\n \\n    frame = image.read()\\n    if frame is not None:  # add this line\\n       #print(frame)\\n      (H, W) = frame.shape[:2]'"
      ]
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "cap = cv2.VideoCapture(0)\r\n",
    "\r\n",
    "count = 0\r\n",
    "while True:\r\n",
    "    #record\r\n",
    "    _, frame = cap.read()\r\n",
    "    #print(frame)\r\n",
    "    frame = imutils.resize(frame, width=600)\r\n",
    "\r\n",
    "    height,width,color = frame.shape\r\n",
    "\r\n",
    "    # frame to grayscale format\r\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\r\n",
    "    faces = detector(gray, 1)    \r\n",
    "    x, y, w, h = 0, 0, 0, 0\r\n",
    "    \r\n",
    "    key = cv2.waitKey(1)\r\n",
    "    \r\n",
    "    text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,255,0), 2)\r\n",
    "    # record frames\r\n",
    "    if recording and len(faces)==1:\r\n",
    "        text = \"Recording... \"\r\n",
    "        text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\r\n",
    "        #loop coordinates\r\n",
    "        for face in faces:\r\n",
    "            # args: image, bounding box of the image\r\n",
    "            landmarks = predictor(gray, face)\r\n",
    "            text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\r\n",
    "        \r\n",
    "            landmarks = face_utils.shape_to_np(landmarks)\r\n",
    "            name, i, j = 'mouth', 48, 68\r\n",
    "        \r\n",
    "        margin_x,margin_y = 5,7        \r\n",
    "        (x, y, w, h) = cv2.boundingRect(np.array([landmarks[i:j]])) \r\n",
    "        \r\n",
    "        # roi \r\n",
    "        roi = gray[y:y+h, x:x+w]\r\n",
    "        roi = imutils.resize(roi, width = 250, inter=cv2.INTER_CUBIC) \r\n",
    "        \r\n",
    "        cv2.rectangle(frame, (x, y), (x+w ,y+h),(0,255,0),2,-1)\r\n",
    "\r\n",
    "#         cv2.imwrite(\"new_dataset/color%d.jpg\" % count, roi)\r\n",
    "        frames.append(np.array(roi))\r\n",
    "        \r\n",
    "        count+=1\r\n",
    "        \r\n",
    "    elif len(frames) > 0:\r\n",
    "        sequence = []\r\n",
    "        count = 0\r\n",
    "        for seq in frames:\r\n",
    "            frame_rz = resize(seq, (100,100))\r\n",
    "            frame_rz = 255 * frame_rz\r\n",
    "            frame_rz = frame_rz.astype(np.uint8)\r\n",
    "            sequence.append(frame_rz)\r\n",
    "        pad_array = [np.zeros((100, 100))]                            \r\n",
    "        sequence.extend(pad_array * (22 - len(sequence)))\r\n",
    "        sequence = np.array(sequence)\r\n",
    "            \r\n",
    "        #normalize\r\n",
    "        np.seterr(divide='ignore', invalid='ignore') # ignore divide by 0 warning\r\n",
    "        v_min = sequence.min(axis=(1, 2), keepdims=True)\r\n",
    "        v_max = sequence.max(axis=(1, 2), keepdims=True)\r\n",
    "        sequence = (sequence - v_min)/(v_max - v_min)\r\n",
    "        sequence = np.nan_to_num(sequence)\r\n",
    "            \r\n",
    "        #my_pred = sequence.reshape(1,22,100,100,1)\r\n",
    "\r\n",
    "        print(sequence.shape)\r\n",
    "\r\n",
    "        img = cv2.resize(sequence,(22,100))\r\n",
    "        my_pred = np.reshape(img,[1,22,100,100,1])\r\n",
    "\r\n",
    "        ans = model.predict(my_pred)\r\n",
    "        percent = round(np.max(ans)*100,2) \r\n",
    "        max_index = np.argmax(ans,)\r\n",
    "        text = \"Predicted : \" + words[max_index] + \" , \" + str(percent) + \" %\" \r\n",
    "        text_display = cv2.putText(frame, text, (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,255,0), 2)\r\n",
    "        frames = []\r\n",
    "    else:\r\n",
    "        recording = False\r\n",
    "        \r\n",
    "    cv2.imshow(\"ALR, space- record, stop\",frame)\r\n",
    "    \r\n",
    "    # Escape key, close cam\r\n",
    "    if key ==27:\r\n",
    "        cap.release()\r\n",
    "        cv2.destroyAllWindows()\r\n",
    "        break\r\n",
    "    # Spacebar , stop and record\r\n",
    "    elif key == 32:\r\n",
    "        if recording == True:\r\n",
    "            recording = False\r\n",
    "        else:\r\n",
    "            recording = True\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(22, 100, 100)\n",
      "(25, 100, 100)\n",
      "(34, 100, 100)\n",
      "(48, 100, 100)\n",
      "(25, 100, 100)\n",
      "(40, 100, 100)\n",
      "(27, 100, 100)\n",
      "(28, 100, 100)\n",
      "(34, 100, 100)\n",
      "(29, 100, 100)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "#jj"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "\"\"\"cap = cv2.VideoCapture(0)\r\n",
    "\r\n",
    "count = 0\r\n",
    "while True:\r\n",
    "#while(cap.isOpened()):\r\n",
    "    #record\r\n",
    "    _, frame = cap.read()\r\n",
    "    frame = imutils.resize(frame, width=600)\r\n",
    "    if not ret:\r\n",
    "        logging.info('Failed to read video')\r\n",
    "        sys.exit(0)\r\n",
    "    image = cv2.resize(image, None, fx=scale, fy=scale)\r\n",
    "    cv2.imshow(\"Original\", image)\r\n",
    "\r\n",
    "\r\n",
    "    #frame_1 = np.array(frame)\r\n",
    "    #print(frame)\r\n",
    "    #height,width,color = frame.shape\r\n",
    "\r\n",
    "    # frame to grayscale format\r\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\r\n",
    "    faces = detector(gray, 1)    \r\n",
    "\r\n",
    "    x, y, w, h = 0, 0, 0, 0\r\n",
    "    \r\n",
    "    key = cv2.waitKey(1)\r\n",
    "    \r\n",
    "    text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(255,255,255), 2)\r\n",
    "    # record frames\r\n",
    "    if recording:\r\n",
    "        text = \"Recording... \"\r\n",
    "        text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\r\n",
    "        #loop coordinates\r\n",
    "        for face in faces:\r\n",
    "            # args: image, bounding box of the image\r\n",
    "            landmarks = predictor(gray, face)\r\n",
    "            text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\r\n",
    "        \r\n",
    "            landmarks = face_utils.shape_to_np(landmarks)\r\n",
    "            name, i, j = 'mouth', 48, 68\r\n",
    "        \r\n",
    "        margin_x,margin_y = 5,7        \r\n",
    "        (x, y, w, h) = cv2.boundingRect(np.array([landmarks[i:j]])) \r\n",
    "        \r\n",
    "        # roi \r\n",
    "        roi = gray[y:y+h, x:x+w]\r\n",
    "        roi = imutils.resize(roi, width = 250, inter=cv2.INTER_CUBIC) \r\n",
    "        \r\n",
    "        cv2.rectangle(frame, (x, y), (x+w ,y+h),(0,255,0),2,-1)\r\n",
    "\r\n",
    "#         cv2.imwrite(\"new_dataset/color%d.jpg\" % count, roi)\r\n",
    "        frames.append(np.array(roi))\r\n",
    "        \r\n",
    "        count+=1\r\n",
    "        \r\n",
    "    elif len(frames) > 0:\r\n",
    "        sequence = []\r\n",
    "        count = 0\r\n",
    "        for frame in frames:\r\n",
    "            frame_rz = resize(frame, (100,100))\r\n",
    "            frame_rz = 255 * frame_rz\r\n",
    "            frame_rz = frame_rz.astype(np.uint8)\r\n",
    "            sequence.append(frame_rz)\r\n",
    "        pad_array = [np.zeros((100, 100))]                            \r\n",
    "        sequence.extend(pad_array * (22 - len(sequence)))\r\n",
    "        sequence = np.array(sequence)\r\n",
    "            \r\n",
    "        #normalize\r\n",
    "        np.seterr(divide='ignore', invalid='ignore') # ignore divide by 0 warning\r\n",
    "        v_min = sequence.min(axis=(1, 2), keepdims=True)\r\n",
    "        v_max = sequence.max(axis=(1, 2), keepdims=True)\r\n",
    "        sequence = (sequence - v_min)/(v_max - v_min)\r\n",
    "        sequence = np.nan_to_num(sequence)\r\n",
    "            \r\n",
    "        my_pred = sequence.reshape(1,22,100,100,1)\r\n",
    "        ans = model.predict(my_pred)\r\n",
    "        percent = round(np.max(ans)*100,2) \r\n",
    "        max_index = np.argmax(ans,)\r\n",
    "        text = \"Predicted : \" + words[max_index] + \" , \" + str(percent) + \" %\" \r\n",
    "        text_display = cv2.putText(frame, text, (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(255,255,255), 2)\r\n",
    "        frames = []\r\n",
    "        \r\n",
    "    cv2.imshow(\"ALR, space- record, stop\",frame)\r\n",
    "    \r\n",
    "    # Escape key, close cam\r\n",
    "    if key ==27:\r\n",
    "        cap.release()\r\n",
    "        cv2.destroyAllWindows()\r\n",
    "        break\r\n",
    "    # Spacebar , stop and record\r\n",
    "    elif key == 32:\r\n",
    "        if recording == True:\r\n",
    "            recording = False\r\n",
    "        else:\r\n",
    "            recording = True\r\n",
    "\r\n",
    "\"\"\""
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'cap = cv2.VideoCapture(0)\\n\\ncount = 0\\nwhile True:\\n#while(cap.isOpened()):\\n    #record\\n    _, frame = cap.read()\\n    frame = imutils.resize(frame, width=600)\\n    if not ret:\\n        logging.info(\\'Failed to read video\\')\\n        sys.exit(0)\\n    image = cv2.resize(image, None, fx=scale, fy=scale)\\n    cv2.imshow(\"Original\", image)\\n\\n\\n    #frame_1 = np.array(frame)\\n    #print(frame)\\n    #height,width,color = frame.shape\\n\\n    # frame to grayscale format\\n    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\\n    faces = detector(gray, 1)    \\n\\n    x, y, w, h = 0, 0, 0, 0\\n    \\n    key = cv2.waitKey(1)\\n    \\n    text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(255,255,255), 2)\\n    # record frames\\n    if recording:\\n        text = \"Recording... \"\\n        text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\\n        #loop coordinates\\n        for face in faces:\\n            # args: image, bounding box of the image\\n            landmarks = predictor(gray, face)\\n            text_display = cv2.putText(frame, text , (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255), 2)\\n        \\n            landmarks = face_utils.shape_to_np(landmarks)\\n            name, i, j = \\'mouth\\', 48, 68\\n        \\n        margin_x,margin_y = 5,7        \\n        (x, y, w, h) = cv2.boundingRect(np.array([landmarks[i:j]])) \\n        \\n        # roi \\n        roi = gray[y:y+h, x:x+w]\\n        roi = imutils.resize(roi, width = 250, inter=cv2.INTER_CUBIC) \\n        \\n        cv2.rectangle(frame, (x, y), (x+w ,y+h),(0,255,0),2,-1)\\n\\n#         cv2.imwrite(\"new_dataset/color%d.jpg\" % count, roi)\\n        frames.append(np.array(roi))\\n        \\n        count+=1\\n        \\n    elif len(frames) > 0:\\n        sequence = []\\n        count = 0\\n        for frame in frames:\\n            frame_rz = resize(frame, (100,100))\\n            frame_rz = 255 * frame_rz\\n            frame_rz = frame_rz.astype(np.uint8)\\n            sequence.append(frame_rz)\\n        pad_array = [np.zeros((100, 100))]                            \\n        sequence.extend(pad_array * (22 - len(sequence)))\\n        sequence = np.array(sequence)\\n            \\n        #normalize\\n        np.seterr(divide=\\'ignore\\', invalid=\\'ignore\\') # ignore divide by 0 warning\\n        v_min = sequence.min(axis=(1, 2), keepdims=True)\\n        v_max = sequence.max(axis=(1, 2), keepdims=True)\\n        sequence = (sequence - v_min)/(v_max - v_min)\\n        sequence = np.nan_to_num(sequence)\\n            \\n        my_pred = sequence.reshape(1,22,100,100,1)\\n        ans = model.predict(my_pred)\\n        percent = round(np.max(ans)*100,2) \\n        max_index = np.argmax(ans,)\\n        text = \"Predicted : \" + words[max_index] + \" , \" + str(percent) + \" %\" \\n        text_display = cv2.putText(frame, text, (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(255,255,255), 2)\\n        frames = []\\n        \\n    cv2.imshow(\"ALR, space- record, stop\",frame)\\n    \\n    # Escape key, close cam\\n    if key ==27:\\n        cap.release()\\n        cv2.destroyAllWindows()\\n        break\\n    # Spacebar , stop and record\\n    elif key == 32:\\n        if recording == True:\\n            recording = False\\n        else:\\n            recording = True\\n\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.5",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit (conda)"
  },
  "interpreter": {
   "hash": "b64de0ba76e769a7c3088015f258876bdb4240a199c36c507f14d2fb10bd5d74"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}