18.2 Building an Emotion Detector with LittleVGG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using LittleVGG for Emotion Detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Emotion Detector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 28273 images belonging to 6 classes.\n",
      "Found 3534 images belonging to 6 classes.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "import os\n",
    "\n",
    "num_classes = 6\n",
    "img_rows, img_cols = 48, 48\n",
    "batch_size = 16\n",
    "\n",
    "train_data_dir = './fer2013/train'\n",
    "validation_data_dir = './fer2013/validation'\n",
    "\n",
    "# Let's use some data augmentaiton \n",
    "train_datagen = ImageDataGenerator(\n",
    "      rescale=1./255,\n",
    "      rotation_range=30,\n",
    "      shear_range=0.3,\n",
    "      zoom_range=0.3,\n",
    "      width_shift_range=0.4,\n",
    "      height_shift_range=0.4,\n",
    "      horizontal_flip=True,\n",
    "      fill_mode='nearest')\n",
    " \n",
    "validation_datagen = ImageDataGenerator(rescale=1./255)\n",
    " \n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "        train_data_dir,\n",
    "        color_mode = 'grayscale',\n",
    "        target_size=(img_rows, img_cols),\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical',\n",
    "        shuffle=True)\n",
    " \n",
    "validation_generator = validation_datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        color_mode = 'grayscale',\n",
    "        target_size=(img_rows, img_cols),\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical',\n",
    "        shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Our Keras Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.layers.advanced_activations import ELU\n",
    "from keras.layers.core import Activation, Flatten, Dropout, Dense"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras LittleVGG Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 48, 48, 32)        320       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 48, 48, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 48, 48, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 48, 48, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 48, 48, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 48, 48, 32)        128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 24, 24, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 24, 24, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 24, 24, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 24, 24, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 24, 24, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 24, 24, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 24, 24, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 24, 24, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 12, 12, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 12, 12, 128)       0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 12, 12, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 12, 12, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 12, 12, 128)       0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 12, 12, 128)       512       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 6, 6, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 6, 6, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 6, 6, 256)         295168    \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 6, 6, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 6, 6, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, 6, 6, 256)         590080    \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 6, 6, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 6, 6, 256)         1024      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 3, 3, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 3, 3, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                147520    \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 6)                 390       \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 6)                 0         \n",
      "=================================================================\n",
      "Total params: 1,328,102\n",
      "Trainable params: 1,325,926\n",
      "Non-trainable params: 2,176\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(32, (3, 3), padding = 'same', kernel_initializer=\"he_normal\",\n",
    "                 input_shape = (img_rows, img_cols, 1)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3, 3), padding = \"same\", kernel_initializer=\"he_normal\", \n",
    "                 input_shape = (img_rows, img_cols, 1)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "# Block #2: second CONV => RELU => CONV => RELU => POOL\n",
    "# layer set\n",
    "model.add(Conv2D(64, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "# Block #3: third CONV => RELU => CONV => RELU => POOL\n",
    "# layer set\n",
    "model.add(Conv2D(128, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "# Block #4: third CONV => RELU => CONV => RELU => POOL\n",
    "# layer set\n",
    "model.add(Conv2D(256, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(256, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "# Block #5: first set of FC => RELU layers\n",
    "model.add(Flatten())\n",
    "model.add(Dense(64, kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('elu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "# Block #6: second set of FC => RELU layers\n",
    "model.add(Dense(64, kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation('elu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "# Block #7: softmax classifier\n",
    "model.add(Dense(num_classes, kernel_initializer=\"he_normal\"))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1767/1767 [==============================] - 977s 553ms/step - loss: 1.9719 - acc: 0.2118 - val_loss: 1.7343 - val_acc: 0.2540\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.73434, saving model to /home/deeplearningcv/DeepLearningCV/Trained Models/emotion_little_vgg_3.h5\n",
      "Epoch 2/10\n",
      "1767/1767 [==============================] - 907s 513ms/step - loss: 1.7441 - acc: 0.2486 - val_loss: 1.7117 - val_acc: 0.2766\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.73434 to 1.71173, saving model to /home/deeplearningcv/DeepLearningCV/Trained Models/emotion_little_vgg_3.h5\n",
      "Epoch 3/10\n",
      " 659/1767 [==========>...................] - ETA: 1:24:41 - loss: 1.7353 - acc: 0.2570"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import RMSprop, SGD, Adam\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n",
    "\n",
    "                     \n",
    "checkpoint = ModelCheckpoint(\"/home/deeplearningcv/DeepLearningCV/Trained Models/emotion_little_vgg_3.h5\",\n",
    "                             monitor=\"val_loss\",\n",
    "                             mode=\"min\",\n",
    "                             save_best_only = True,\n",
    "                             verbose=1)\n",
    "\n",
    "earlystop = EarlyStopping(monitor = 'val_loss', \n",
    "                          min_delta = 0, \n",
    "                          patience = 3,\n",
    "                          verbose = 1,\n",
    "                          restore_best_weights = True)\n",
    "\n",
    "reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, min_delta = 0.0001)\n",
    "\n",
    "# we put our call backs into a callback list\n",
    "callbacks = [earlystop, checkpoint, reduce_lr]\n",
    "\n",
    "# We use a very small learning rate \n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = Adam(lr=0.001),\n",
    "              metrics = ['accuracy'])\n",
    "\n",
    "nb_train_samples = 28273\n",
    "nb_validation_samples = 3534\n",
    "epochs = 10\n",
    "\n",
    "history = model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch = nb_train_samples // batch_size,\n",
    "    epochs = epochs,\n",
    "    callbacks = callbacks,\n",
    "    validation_data = validation_generator,\n",
    "    validation_steps = nb_validation_samples // batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "nb_train_samples = 28273\n",
    "nb_validation_samples = 3534\n",
    "\n",
    "# We need to recreate our validation generator with shuffle = false\n",
    "validation_generator = validation_datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        color_mode = 'grayscale',\n",
    "        target_size=(img_rows, img_cols),\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical',\n",
    "        shuffle=False)\n",
    "\n",
    "class_labels = validation_generator.class_indices\n",
    "class_labels = {v: k for k, v in class_labels.items()}\n",
    "classes = list(class_labels.values())\n",
    "\n",
    "#Confution Matrix and Classification Report\n",
    "Y_pred = model.predict_generator(validation_generator, nb_validation_samples // batch_size+1)\n",
    "y_pred = np.argmax(Y_pred, axis=1)\n",
    "\n",
    "print('Confusion Matrix')\n",
    "print(confusion_matrix(validation_generator.classes, y_pred))\n",
    "print('Classification Report')\n",
    "target_names = list(class_labels.values())\n",
    "print(classification_report(validation_generator.classes, y_pred, target_names=target_names))\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "cnf_matrix = confusion_matrix(validation_generator.classes, y_pred)\n",
    "\n",
    "plt.imshow(cnf_matrix, interpolation='nearest')\n",
    "plt.colorbar()\n",
    "tick_marks = np.arange(len(classes))\n",
    "_ = plt.xticks(tick_marks, classes, rotation=90)\n",
    "_ = plt.yticks(tick_marks, classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading our saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "classifier = load_model('/home/deeplearningcv/DeepLearningCV/Trained Models/emotion_little_vgg_3.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get our class labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_generator = validation_datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        color_mode = 'grayscale',\n",
    "        target_size=(img_rows, img_cols),\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical',\n",
    "        shuffle=False)\n",
    "\n",
    "class_labels = validation_generator.class_indices\n",
    "class_labels = {v: k for k, v in class_labels.items()}\n",
    "classes = list(class_labels.values())\n",
    "print(class_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's test on some of validation images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deeplearningcv/anaconda3/envs/cv/lib/python3.6/site-packages/keras_preprocessing/image.py:492: UserWarning: grayscale is deprecated. Please use color_mode = \"grayscale\"\n",
      "  warnings.warn('grayscale is deprecated. Please use '\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "from keras.optimizers import RMSprop, SGD, Adam\n",
    "from keras.preprocessing import image\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import re\n",
    "\n",
    "def draw_test(name, pred, im, true_label):\n",
    "    BLACK = [0,0,0]\n",
    "    expanded_image = cv2.copyMakeBorder(im, 160, 0, 0, 300 ,cv2.BORDER_CONSTANT,value=BLACK)\n",
    "    cv2.putText(expanded_image, \"predited - \"+ pred, (20, 60) , cv2.FONT_HERSHEY_SIMPLEX,1, (0,0,255), 2)\n",
    "    cv2.putText(expanded_image, \"true - \"+ true_label, (20, 120) , cv2.FONT_HERSHEY_SIMPLEX,1, (0,255,0), 2)\n",
    "    cv2.imshow(name, expanded_image)\n",
    "\n",
    "\n",
    "def getRandomImage(path, img_width, img_height):\n",
    "    \"\"\"function loads a random images from a random folder in our test path \"\"\"\n",
    "    folders = list(filter(lambda x: os.path.isdir(os.path.join(path, x)), os.listdir(path)))\n",
    "    random_directory = np.random.randint(0,len(folders))\n",
    "    path_class = folders[random_directory]\n",
    "    file_path = path + path_class\n",
    "    file_names = [f for f in listdir(file_path) if isfile(join(file_path, f))]\n",
    "    random_file_index = np.random.randint(0,len(file_names))\n",
    "    image_name = file_names[random_file_index]\n",
    "    final_path = file_path + \"/\" + image_name\n",
    "    return image.load_img(final_path, target_size = (img_width, img_height),grayscale=True), final_path, path_class\n",
    "\n",
    "# dimensions of our images\n",
    "img_width, img_height = 48, 48\n",
    "\n",
    "# We use a very small learning rate \n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = RMSprop(lr = 0.001),\n",
    "              metrics = ['accuracy'])\n",
    "\n",
    "files = []\n",
    "predictions = []\n",
    "true_labels = []\n",
    "\n",
    "# predicting images\n",
    "for i in range(0, 10):\n",
    "    path = './fer2013/validation/' \n",
    "    img, final_path, true_label = getRandomImage(path, img_width, img_height)\n",
    "    files.append(final_path)\n",
    "    true_labels.append(true_label)\n",
    "    x = image.img_to_array(img)\n",
    "    x = x * 1./255\n",
    "    x = np.expand_dims(x, axis=0)\n",
    "    images = np.vstack([x])\n",
    "    classes = model.predict_classes(images, batch_size = 10)\n",
    "    predictions.append(classes)\n",
    "    \n",
    "for i in range(0, len(files)):\n",
    "    image = cv2.imread((files[i]))\n",
    "    image = cv2.resize(image, None, fx=3, fy=3, interpolation = cv2.INTER_CUBIC)\n",
    "    draw_test(\"Prediction\", class_labels[predictions[i][0]], image, true_labels[i])\n",
    "    cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test on a single image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "from keras.preprocessing import image\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from keras.preprocessing.image import img_to_array\n",
    "\n",
    "face_classifier = cv2.CascadeClassifier('./Haarcascades/haarcascade_frontalface_default.xml')\n",
    "\n",
    "def face_detector(img):\n",
    "    # Convert image to grayscale\n",
    "    gray = cv2.cvtColor(img.copy(),cv2.COLOR_BGR2GRAY)\n",
    "    faces = face_classifier.detectMultiScale(gray, 1.3, 5)\n",
    "    if faces is ():\n",
    "        return (0,0,0,0), np.zeros((48,48), np.uint8), img\n",
    "    \n",
    "    allfaces = []   \n",
    "    rects = []\n",
    "    for (x,y,w,h) in faces:\n",
    "        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "        roi_gray = gray[y:y+h, x:x+w]\n",
    "        roi_gray = cv2.resize(roi_gray, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "        allfaces.append(roi_gray)\n",
    "        rects.append((x,w,y,h))\n",
    "    return rects, allfaces, img\n",
    "\n",
    "img = cv2.imread(\"rajeev.jpg\")\n",
    "rects, faces, image = face_detector(img)\n",
    "\n",
    "i = 0\n",
    "for face in faces:\n",
    "    roi = face.astype(\"float\") / 255.0\n",
    "    roi = img_to_array(roi)\n",
    "    roi = np.expand_dims(roi, axis=0)\n",
    "\n",
    "    # make a prediction on the ROI, then lookup the class\n",
    "    preds = classifier.predict(roi)[0]\n",
    "    label = class_labels[preds.argmax()]   \n",
    "\n",
    "    #Overlay our detected emotion on our pic\n",
    "    label_position = (rects[i][0] + int((rects[i][1]/2)), abs(rects[i][2] - 10))\n",
    "    i =+ 1\n",
    "    cv2.putText(image, label, label_position , cv2.FONT_HERSHEY_SIMPLEX,1, (0,255,0), 2)\n",
    "    \n",
    "cv2.imshow(\"Emotion Detector\", image)\n",
    "cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's try this on our webcam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "from time import sleep\n",
    "from keras.preprocessing.image import img_to_array\n",
    "\n",
    "face_classifier = cv2.CascadeClassifier('./Haarcascades/haarcascade_frontalface_default.xml')\n",
    "\n",
    "def face_detector(img):\n",
    "    # Convert image to grayscale\n",
    "    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n",
    "    faces = face_classifier.detectMultiScale(gray, 1.3, 5)\n",
    "    if faces is ():\n",
    "        return (0,0,0,0), np.zeros((48,48), np.uint8), img\n",
    "    \n",
    "    for (x,y,w,h) in faces:\n",
    "        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "        roi_gray = gray[y:y+h, x:x+w]\n",
    "\n",
    "    try:\n",
    "        roi_gray = cv2.resize(roi_gray, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "    except:\n",
    "        return (x,w,y,h), np.zeros((48,48), np.uint8), img\n",
    "    return (x,w,y,h), roi_gray, img\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "while True:\n",
    "\n",
    "    ret, frame = cap.read()\n",
    "    rect, face, image = face_detector(frame)\n",
    "    if np.sum([face]) != 0.0:\n",
    "        roi = face.astype(\"float\") / 255.0\n",
    "        roi = img_to_array(roi)\n",
    "        roi = np.expand_dims(roi, axis=0)\n",
    "\n",
    "        # make a prediction on the ROI, then lookup the class\n",
    "        preds = classifier.predict(roi)[0]\n",
    "        label = class_labels[preds.argmax()]  \n",
    "        label_position = (rect[0] + int((rect[1]/2)), rect[2] + 25)\n",
    "        cv2.putText(image, label, label_position , cv2.FONT_HERSHEY_SIMPLEX,2, (0,255,0), 3)\n",
    "    else:\n",
    "        cv2.putText(image, \"No Face Found\", (20, 60) , cv2.FONT_HERSHEY_SIMPLEX,2, (0,255,0), 3)\n",
    "        \n",
    "    cv2.imshow('All', image)\n",
    "    if cv2.waitKey(1) == 13: #13 is the Enter Key\n",
    "        break\n",
    "        \n",
    "cap.release()\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}