facenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "distance for lokesh is 1.161332\n",
      "distance for lokesh is 1.2331723\n",
      "distance for lokesh is 1.2506509\n",
      "distance for lokesh is 1.2353574\n",
      "distance for lokesh is 1.1652672\n",
      "distance for lokesh is 1.0270791\n",
      "distance for lokesh is 1.0602683\n",
      "distance for lokesh is 1.1110951\n",
      "distance for lokesh is 1.0784804\n",
      "distance for lokesh is 1.2191217\n",
      "distance for lokesh is 1.1955708\n",
      "distance for lokesh is 1.2362756\n",
      "distance for lokesh is 1.1241958\n",
      "distance for lokesh is 1.1417767\n",
      "distance for lokesh is 1.0887704\n",
      "distance for lokesh is 1.0091482\n",
      "distance for lokesh is 0.8991391\n",
      "distance for lokesh is 0.97845376\n",
      "distance for lokesh is 1.0454416\n",
      "distance for lokesh is 0.85601723\n",
      "distance for lokesh is 0.9028682\n",
      "distance for lokesh is 0.90300316\n",
      "distance for lokesh is 1.0018396\n",
      "distance for lokesh is 0.9928967\n",
      "distance for lokesh is 0.9082797\n",
      "distance for lokesh is 0.939407\n",
      "distance for lokesh is 1.0222032\n",
      "distance for lokesh is 0.93216115\n",
      "distance for lokesh is 0.96645135\n",
      "distance for lokesh is 0.9432512\n",
      "distance for lokesh is 0.93972915\n",
      "distance for lokesh is 1.0796149\n",
      "distance for lokesh is 0.9720652\n",
      "distance for lokesh is 1.0925616\n",
      "distance for lokesh is 1.0280339\n",
      "distance for lokesh is 1.0704951\n",
      "distance for lokesh is 1.0925757\n",
      "distance for lokesh is 0.79472965\n",
      "distance for lokesh is 1.066613\n",
      "distance for lokesh is 1.1873624\n",
      "distance for lokesh is 1.0232792\n",
      "distance for lokesh is 0.9475083\n",
      "distance for lokesh is 0.8434777\n",
      "distance for lokesh is 0.9548176\n",
      "distance for lokesh is 0.96026427\n",
      "distance for lokesh is 1.1098489\n",
      "distance for lokesh is 1.2730753\n",
      "distance for lokesh is 1.1830003\n",
      "distance for lokesh is 1.1910522\n",
      "distance for lokesh is 1.1627742\n",
      "distance for lokesh is 1.1372503\n",
      "distance for lokesh is 1.1546247\n",
      "distance for lokesh is 1.1146543\n",
      "distance for lokesh is 1.0763452\n",
      "distance for lokesh is 1.1705178\n",
      "distance for lokesh is 1.1694027\n",
      "distance for lokesh is 1.0793583\n",
      "distance for lokesh is 1.1832787\n",
      "distance for lokesh is 1.1284463\n",
      "distance for lokesh is 1.1196773\n",
      "distance for lokesh is 1.1617084\n",
      "distance for lokesh is 1.0896957\n",
      "distance for lokesh is 1.0146767\n",
      "distance for lokesh is 1.0477779\n",
      "distance for lokesh is 1.1323568\n",
      "distance for lokesh is 1.0155966\n",
      "distance for lokesh is 0.88026994\n",
      "distance for lokesh is 0.9196497\n",
      "distance for lokesh is 0.9497829\n",
      "distance for lokesh is 0.942734\n",
      "distance for lokesh is 0.96706843\n",
      "distance for lokesh is 0.89337766\n",
      "distance for lokesh is 0.8664606\n",
      "distance for lokesh is 0.9428285\n",
      "distance for lokesh is 0.9268734\n",
      "distance for lokesh is 0.8101274\n",
      "distance for lokesh is 0.88755685\n",
      "distance for lokesh is 0.9722703\n",
      "distance for lokesh is 0.9557072\n",
      "distance for lokesh is 0.8647813\n",
      "distance for lokesh is 0.8990592\n",
      "distance for lokesh is 0.8788958\n",
      "distance for lokesh is 0.8672534\n",
      "distance for lokesh is 0.8787259\n",
      "distance for lokesh is 0.8114299\n",
      "distance for lokesh is 0.8400765\n",
      "distance for lokesh is 0.8839062\n",
      "distance for lokesh is 0.8185518\n",
      "distance for lokesh is 0.8678113\n",
      "distance for lokesh is 0.8156619\n",
      "distance for lokesh is 0.93306696\n",
      "distance for lokesh is 0.8373531\n",
      "distance for lokesh is 0.94602424\n",
      "distance for lokesh is 0.8498136\n",
      "distance for lokesh is 0.89338815\n",
      "distance for lokesh is 0.9731251\n",
      "distance for lokesh is 0.95550203\n",
      "distance for lokesh is 0.96286356\n",
      "distance for lokesh is 1.0057846\n",
      "distance for lokesh is 1.0214992\n",
      "distance for lokesh is 0.91720307\n",
      "distance for lokesh is 1.127044\n",
      "distance for lokesh is 1.1179944\n",
      "distance for lokesh is 1.0769292\n",
      "distance for lokesh is 1.0587294\n",
      "distance for lokesh is 0.98094815\n",
      "distance for lokesh is 0.9828265\n",
      "distance for lokesh is 1.0953301\n",
      "distance for lokesh is 1.113157\n",
      "distance for lokesh is 1.0206752\n",
      "distance for lokesh is 1.0380067\n",
      "distance for lokesh is 1.0898741\n",
      "distance for lokesh is 0.8733914\n",
      "distance for lokesh is 0.87693554\n",
      "distance for lokesh is 0.8621443\n",
      "distance for lokesh is 0.8196437\n",
      "distance for lokesh is 0.8828073\n",
      "distance for lokesh is 0.99496603\n",
      "distance for lokesh is 0.81840295\n",
      "distance for lokesh is 0.6178533\n",
      "distance for lokesh is 0.683306\n",
      "distance for lokesh is 0.7411224\n",
      "distance for lokesh is 0.6877366\n",
      "distance for lokesh is 0.8248519\n",
      "distance for lokesh is 0.86548746\n",
      "distance for lokesh is 0.9230631\n",
      "distance for lokesh is 0.95463264\n",
      "distance for lokesh is 1.0288948\n",
      "distance for lokesh is 1.0414178\n",
      "distance for lokesh is 1.1767802\n",
      "distance for lokesh is 1.0954616\n",
      "distance for lokesh is 1.0665078\n",
      "distance for lokesh is 1.1866\n",
      "distance for lokesh is 1.2410579\n",
      "distance for lokesh is 1.2734709\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "import time\n",
    "from multiprocessing.dummy import Pool\n",
    "K.set_image_data_format('channels_first')\n",
    "import cv2\n",
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "from numpy import genfromtxt\n",
    "import tensorflow as tf\n",
    "from fr_utils import *\n",
    "from inception_blocks_v2 import *\n",
    "import win32com.client as wincl\n",
    "\n",
    "PADDING = 50\n",
    "ready_to_detect_identity = True\n",
    "windows10_voice_interface = wincl.Dispatch(\"SAPI.SpVoice\")\n",
    "\n",
    "FRmodel = faceRecoModel(input_shape=(3, 96, 96))\n",
    "\n",
    "def triplet_loss(y_true, y_pred, alpha = 0.3):\n",
    "    \"\"\"\n",
    "    Implementation of the triplet loss as defined by formula (3)\n",
    "    \n",
    "    Arguments:\n",
    "    y_pred -- python list containing three objects:\n",
    "            anchor -- the encodings for the anchor images, of shape (None, 128)\n",
    "            positive -- the encodings for the positive images, of shape (None, 128)\n",
    "            negative -- the encodings for the negative images, of shape (None, 128)\n",
    "    \n",
    "    Returns:\n",
    "    loss -- real number, value of the loss\n",
    "    \"\"\"\n",
    "    \n",
    "    anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]\n",
    "    \n",
    "    # Step 1: Compute the (encoding) distance between the anchor and the positive, you will need to sum over axis=-1\n",
    "    pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), axis=-1)\n",
    "    # Step 2: Compute the (encoding) distance between the anchor and the negative, you will need to sum over axis=-1\n",
    "    neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), axis=-1)\n",
    "    # Step 3: subtract the two previous distances and add alpha.\n",
    "    basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), alpha)\n",
    "    # Step 4: Take the maximum of basic_loss and 0.0. Sum over the training examples.\n",
    "    loss = tf.reduce_sum(tf.maximum(basic_loss, 0.0))\n",
    "    \n",
    "    return loss\n",
    "\n",
    "FRmodel.compile(optimizer = 'adam', loss = triplet_loss, metrics = ['accuracy'])\n",
    "load_weights_from_FaceNet(FRmodel)\n",
    "\n",
    "def prepare_database():\n",
    "    database = {}\n",
    "\n",
    "    # load all the images of individuals to recognize into the database\n",
    "    for file in glob.glob(\"images/*\"):\n",
    "        identity = os.path.splitext(os.path.basename(file))[0]\n",
    "        database[identity] = img_path_to_encoding(file, FRmodel)\n",
    "\n",
    "    return database\n",
    "\n",
    "def webcam_face_recognizer(database):\n",
    "    \"\"\"\n",
    "    Runs a loop that extracts images from the computer's webcam and determines whether or not\n",
    "    it contains the face of a person in our database.\n",
    "\n",
    "    If it contains a face, an audio message will be played welcoming the user.\n",
    "    If not, the program will process the next frame from the webcam\n",
    "    \"\"\"\n",
    "    global ready_to_detect_identity\n",
    "\n",
    "    cv2.namedWindow(\"preview\")\n",
    "    vc = cv2.VideoCapture(0)\n",
    "\n",
    "    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')\n",
    "    \n",
    "    while vc.isOpened():\n",
    "        _, frame = vc.read()\n",
    "        img = frame\n",
    "\n",
    "        # We do not want to detect a new identity while the program is in the process of identifying another person\n",
    "        if ready_to_detect_identity:\n",
    "            img = process_frame(img, frame, face_cascade)   \n",
    "        \n",
    "        key = cv2.waitKey(100)\n",
    "        cv2.imshow(\"preview\", img)\n",
    "\n",
    "        if key == 27: # exit on ESC\n",
    "            break\n",
    "    cv2.destroyWindow(\"preview\")\n",
    "\n",
    "def process_frame(img, frame, face_cascade):\n",
    "    \"\"\"\n",
    "    Determine whether the current frame contains the faces of people from our database\n",
    "    \"\"\"\n",
    "    global ready_to_detect_identity\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "    faces = face_cascade.detectMultiScale(gray, 1.3, 5)\n",
    "\n",
    "    # Loop through all the faces detected and determine whether or not they are in the database\n",
    "    identities = []\n",
    "    for (x, y, w, h) in faces:\n",
    "        x1 = x-PADDING\n",
    "        y1 = y-PADDING\n",
    "        x2 = x+w+PADDING\n",
    "        y2 = y+h+PADDING\n",
    "\n",
    "        img = cv2.rectangle(frame,(x1, y1),(x2, y2),(255,0,0),2)\n",
    "\n",
    "        identity = find_identity(frame, x1, y1, x2, y2)\n",
    "\n",
    "        if identity is not None:\n",
    "            identities.append(identity)\n",
    "\n",
    "    if identities != []:\n",
    "        cv2.imwrite('example.png',img)\n",
    "\n",
    "        ready_to_detect_identity = False\n",
    "        pool = Pool(processes=1) \n",
    "        # We run this as a separate process so that the camera feedback does not freeze\n",
    "        pool.apply_async(welcome_users, [identities])\n",
    "    return img\n",
    "\n",
    "def find_identity(frame, x1, y1, x2, y2):\n",
    "    \"\"\"\n",
    "    Determine whether the face contained within the bounding box exists in our database\n",
    "\n",
    "    x1,y1_____________\n",
    "    |                 |\n",
    "    |                 |\n",
    "    |_________________x2,y2\n",
    "\n",
    "    \"\"\"\n",
    "    height, width, channels = frame.shape\n",
    "    # The padding is necessary since the OpenCV face detector creates the bounding box around the face and not the head\n",
    "    part_image = frame[max(0, y1):min(height, y2), max(0, x1):min(width, x2)]\n",
    "    \n",
    "    return who_is_it(part_image, database, FRmodel)\n",
    "\n",
    "def who_is_it(image, database, model):\n",
    "    \"\"\"\n",
    "    Implements face recognition for the happy house by finding who is the person on the image_path image.\n",
    "    \n",
    "    Arguments:\n",
    "    image_path -- path to an image\n",
    "    database -- database containing image encodings along with the name of the person on the image\n",
    "    model -- your Inception model instance in Keras\n",
    "    \n",
    "    Returns:\n",
    "    min_dist -- the minimum distance between image_path encoding and the encodings from the database\n",
    "    identity -- string, the name prediction for the person on image_path\n",
    "    \"\"\"\n",
    "    encoding = img_to_encoding(image, model)\n",
    "    \n",
    "    min_dist = 100\n",
    "    identity = None\n",
    "    \n",
    "    # Loop over the database dictionary's names and encodings.\n",
    "    for (name, db_enc) in database.items():\n",
    "        \n",
    "        # Compute L2 distance between the target \"encoding\" and the current \"emb\" from the database.\n",
    "        dist = np.linalg.norm(db_enc - encoding)\n",
    "\n",
    "        print('distance for %s is %s' %(name, dist))\n",
    "\n",
    "        # If this distance is less than the min_dist, then set min_dist to dist, and identity to name\n",
    "        if dist < min_dist:\n",
    "            min_dist = dist\n",
    "            identity = name\n",
    "    \n",
    "    if min_dist > 0.52:\n",
    "        return None\n",
    "    else:\n",
    "        return str(identity)\n",
    "\n",
    "def welcome_users(identities):\n",
    "    \"\"\" Outputs a welcome audio message to the users \"\"\"\n",
    "    global ready_to_detect_identity\n",
    "    welcome_message = 'Welcome '\n",
    "\n",
    "    if len(identities) == 1:\n",
    "        welcome_message += '%s, have a nice day.' % identities[0]\n",
    "    else:\n",
    "        for identity_id in range(len(identities)-1):\n",
    "            welcome_message += '%s, ' % identities[identity_id]\n",
    "        welcome_message += 'and %s, ' % identities[-1]\n",
    "        welcome_message += 'have a nice day!'\n",
    "\n",
    "    windows10_voice_interface.Speak(welcome_message)\n",
    "\n",
    "    # Allow the program to start detecting identities again\n",
    "    ready_to_detect_identity = True\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    database = prepare_database()\n",
    "    webcam_face_recognizer(database)\n",
    "\n",
    "# ### References:\n",
    "# \n",
    "# - Florian Schroff, Dmitry Kalenichenko, James Philbin (2015). [FaceNet: A Unified Embedding for Face Recognition and Clustering](https://arxiv.org/pdf/1503.03832.pdf)\n",
    "# - Yaniv Taigman, Ming Yang, Marc'Aurelio Ranzato, Lior Wolf (2014). [DeepFace: Closing the gap to human-level performance in face verification](https://research.fb.com/wp-content/uploads/2016/11/deepface-closing-the-gap-to-human-level-performance-in-face-verification.pdf) \n",
    "# - The pretrained model we use is inspired by Victor Sy Wang's implementation and was loaded using his code: https://github.com/iwantooxxoox/Keras-OpenFace.\n",
    "# - Our implementation also took a lot of inspiration from the official FaceNet github repository: https://github.com/davidsandberg/facenet \n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}