Recurrent_Neural_Network_Step_by_step.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Recurrent Neural Network - Step by step",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DhamuSniper/Natural-Language-Processing---Sequence-Data-world/blob/master/Recurrent_Neural_Network_Step_by_step.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yQYSGtGq9FTw",
        "colab_type": "text"
      },
      "source": [
        "Importing essentials and updating RNN utils\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nRvVbU5PpzUm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "def softmax(x):\n",
        "    e_x = np.exp(x - np.max(x))\n",
        "    return e_x / e_x.sum(axis=0)\n",
        "\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1 / (1 + np.exp(-x))\n",
        "\n",
        "\n",
        "def initialize_adam(parameters) :\n",
        "    \"\"\"\n",
        "    Initializes v and s as two python dictionaries with:\n",
        "                - keys: \"dW1\", \"db1\", ..., \"dWL\", \"dbL\" \n",
        "                - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters.\n",
        "\n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters.\n",
        "                    parameters[\"W\" + str(l)] = Wl\n",
        "                    parameters[\"b\" + str(l)] = bl\n",
        "\n",
        "    Returns: \n",
        "    v -- python dictionary that will contain the exponentially weighted average of the gradient.\n",
        "                    v[\"dW\" + str(l)] = ...\n",
        "                    v[\"db\" + str(l)] = ...\n",
        "    s -- python dictionary that will contain the exponentially weighted average of the squared gradient.\n",
        "                    s[\"dW\" + str(l)] = ...\n",
        "                    s[\"db\" + str(l)] = ...\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    L = len(parameters) // 2 # number of layers in the neural networks\n",
        "    v = {}\n",
        "    s = {}\n",
        "\n",
        "    # Initialize v, s. Input: \"parameters\". Outputs: \"v, s\".\n",
        "    for l in range(L):\n",
        "    ### START CODE HERE ### (approx. 4 lines)\n",
        "        v[\"dW\" + str(l+1)] = np.zeros(parameters[\"W\" + str(l+1)].shape)\n",
        "        v[\"db\" + str(l+1)] = np.zeros(parameters[\"b\" + str(l+1)].shape)\n",
        "        s[\"dW\" + str(l+1)] = np.zeros(parameters[\"W\" + str(l+1)].shape)\n",
        "        s[\"db\" + str(l+1)] = np.zeros(parameters[\"b\" + str(l+1)].shape)\n",
        "    ### END CODE HERE ###\n",
        "\n",
        "    return v, s\n",
        "\n",
        "\n",
        "def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate = 0.01,\n",
        "                                beta1 = 0.9, beta2 = 0.999,  epsilon = 1e-8):\n",
        "    \"\"\"\n",
        "    Update parameters using Adam\n",
        "\n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters:\n",
        "                    parameters['W' + str(l)] = Wl\n",
        "                    parameters['b' + str(l)] = bl\n",
        "    grads -- python dictionary containing your gradients for each parameters:\n",
        "                    grads['dW' + str(l)] = dWl\n",
        "                    grads['db' + str(l)] = dbl\n",
        "    v -- Adam variable, moving average of the first gradient, python dictionary\n",
        "    s -- Adam variable, moving average of the squared gradient, python dictionary\n",
        "    learning_rate -- the learning rate, scalar.\n",
        "    beta1 -- Exponential decay hyperparameter for the first moment estimates \n",
        "    beta2 -- Exponential decay hyperparameter for the second moment estimates \n",
        "    epsilon -- hyperparameter preventing division by zero in Adam updates\n",
        "\n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters \n",
        "    v -- Adam variable, moving average of the first gradient, python dictionary\n",
        "    s -- Adam variable, moving average of the squared gradient, python dictionary\n",
        "    \"\"\"\n",
        "\n",
        "    L = len(parameters) // 2                 # number of layers in the neural networks\n",
        "    v_corrected = {}                         # Initializing first moment estimate, python dictionary\n",
        "    s_corrected = {}                         # Initializing second moment estimate, python dictionary\n",
        "\n",
        "    # Perform Adam update on all parameters\n",
        "    for l in range(L):\n",
        "        # Moving average of the gradients. Inputs: \"v, grads, beta1\". Output: \"v\".\n",
        "        ### START CODE HERE ### (approx. 2 lines)\n",
        "        v[\"dW\" + str(l+1)] = beta1 * v[\"dW\" + str(l+1)] + (1 - beta1) * grads[\"dW\" + str(l+1)] \n",
        "        v[\"db\" + str(l+1)] = beta1 * v[\"db\" + str(l+1)] + (1 - beta1) * grads[\"db\" + str(l+1)] \n",
        "        ### END CODE HERE ###\n",
        "\n",
        "        # Compute bias-corrected first moment estimate. Inputs: \"v, beta1, t\". Output: \"v_corrected\".\n",
        "        ### START CODE HERE ### (approx. 2 lines)\n",
        "        v_corrected[\"dW\" + str(l+1)] = v[\"dW\" + str(l+1)] / (1 - beta1**t)\n",
        "        v_corrected[\"db\" + str(l+1)] = v[\"db\" + str(l+1)] / (1 - beta1**t)\n",
        "        ### END CODE HERE ###\n",
        "\n",
        "        # Moving average of the squared gradients. Inputs: \"s, grads, beta2\". Output: \"s\".\n",
        "        ### START CODE HERE ### (approx. 2 lines)\n",
        "        s[\"dW\" + str(l+1)] = beta2 * s[\"dW\" + str(l+1)] + (1 - beta2) * (grads[\"dW\" + str(l+1)] ** 2)\n",
        "        s[\"db\" + str(l+1)] = beta2 * s[\"db\" + str(l+1)] + (1 - beta2) * (grads[\"db\" + str(l+1)] ** 2)\n",
        "        ### END CODE HERE ###\n",
        "\n",
        "        # Compute bias-corrected second raw moment estimate. Inputs: \"s, beta2, t\". Output: \"s_corrected\".\n",
        "        ### START CODE HERE ### (approx. 2 lines)\n",
        "        s_corrected[\"dW\" + str(l+1)] = s[\"dW\" + str(l+1)] / (1 - beta2 ** t)\n",
        "        s_corrected[\"db\" + str(l+1)] = s[\"db\" + str(l+1)] / (1 - beta2 ** t)\n",
        "        ### END CODE HERE ###\n",
        "\n",
        "        # Update parameters. Inputs: \"parameters, learning_rate, v_corrected, s_corrected, epsilon\". Output: \"parameters\".\n",
        "        ### START CODE HERE ### (approx. 2 lines)\n",
        "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * v_corrected[\"dW\" + str(l+1)] / np.sqrt(s_corrected[\"dW\" + str(l+1)] + epsilon)\n",
        "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * v_corrected[\"db\" + str(l+1)] / np.sqrt(s_corrected[\"db\" + str(l+1)] + epsilon)\n",
        "        ### END CODE HERE ###\n",
        "\n",
        "    return parameters, v, s"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HkJ40fis9SOO",
        "colab_type": "text"
      },
      "source": [
        "RNN CELL CREATION"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qvs4SRzqp8aS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def rnn_cell_forward(xt,a_prev,parameters):\n",
        "  \n",
        "  Wax=parameters[\"Wax\"]\n",
        "  Waa=parameters[\"Waa\"]\n",
        "  Wya=parameters[\"Wya\"]\n",
        "  ba=parameters[\"ba\"]\n",
        "  by=parameters[\"by\"]\n",
        "  \n",
        "  #next_Time step activation calculation\n",
        "  a_next=np.tanh(np.dot(Wax,xt)+np.dot(Waa,a_prev)+ba)\n",
        "  \n",
        "  \n",
        "  #prediction of current time step input\n",
        "\n",
        "  yt_pred=softmax(np.dot(Wya,a_next)+by)\n",
        "  \n",
        "  cache = (a_next, a_prev, xt, parameters)\n",
        "\n",
        "  \n",
        "  return a_next,yt_pred, cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llPi_l-qrqz1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "6fc7740c-4da2-4e7f-954f-f6ea43584d8b"
      },
      "source": [
        "np.random.seed(1)\n",
        "xt=np.random.randn(3,10)\n",
        "a_prev=np.random.randn(5,10)\n",
        "Waa=np.random.randn(5,5)\n",
        "Wax=np.random.randn(5,3)\n",
        "Wya=np.random.randn(2,5)\n",
        "ba=np.random.randn(5,1)\n",
        "by=np.random.randn(2,1)\n",
        "\n",
        "parameters={\"Waa\":Waa,\"Wax\":Wax,\"Wya\":Wya,\"ba\":ba,\"by\":by}\n",
        "\n",
        "a_next,yt_pred,cache=rnn_cell_forward(xt,a_prev, parameters)\n",
        "\n",
        "print(\"a_next[4]\",a_next[4])\n",
        "print(\"a_next.shape\",a_next.shape)\n",
        "print(\"yt_pred[1]\",yt_pred[1])\n",
        "print(\"yt_pred.shape\",yt_pred.shape)\n",
        "             "
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "a_next[4] [ 0.59584544  0.18141802  0.61311866  0.99808218  0.85016201  0.99980978\n",
            " -0.18887155  0.99815551  0.6531151   0.82872037]\n",
            "a_next.shape (5, 10)\n",
            "yt_pred[1] [0.9888161  0.01682021 0.21140899 0.36817467 0.98988387 0.88945212\n",
            " 0.36920224 0.9966312  0.9982559  0.17746526]\n",
            "yt_pred.shape (2, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qy1vm8Oa9hhw",
        "colab_type": "text"
      },
      "source": [
        "Forward propagation of RNN cell"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZXVNLQJrvN1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def rnn_forward(x, a0, parameters):\n",
        "  \n",
        "    caches = []\n",
        "    \n",
        "\n",
        "    n_x, m, T_x = x.shape\n",
        "    n_y, n_a = parameters[\"Wya\"].shape\n",
        "    \n",
        "    #initializing with zeros for input with RNN cell\n",
        "    a = np.zeros((n_a, m, T_x))\n",
        "\n",
        "    y_pred =np.zeros((n_y, m, T_x))\n",
        "\n",
        "\n",
        "    a_next =  a0\n",
        "\n",
        "    for t in range(T_x):\n",
        "  \n",
        "        a_next, yt_pred, cache =rnn_cell_forward(x[:,:,t], a_next, parameters)\n",
        "       \n",
        "        a[:,:,t] = a_next\n",
        "        y_pred[:,:,t] = yt_pred\n",
        "        caches.append(cache)\n",
        "\n",
        "    #It stores values for back propagation\n",
        "    caches = (caches, x)\n",
        "    \n",
        "    return a, y_pred, caches"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K1m8UP1I2QSb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "b1205016-e073-4dc9-eafe-5cabd2a524dc"
      },
      "source": [
        "np.random.seed(1)\n",
        "x = np.random.randn(3,10,4)\n",
        "a0 = np.random.randn(5,10)\n",
        "Waa = np.random.randn(5,5)\n",
        "Wax = np.random.randn(5,3)\n",
        "Wya = np.random.randn(2,5)\n",
        "ba = np.random.randn(5,1)\n",
        "by = np.random.randn(2,1)\n",
        "parameters = {\"Waa\": Waa, \"Wax\": Wax, \"Wya\": Wya, \"ba\": ba, \"by\": by}\n",
        "\n",
        "a, y_pred, caches = rnn_forward(x, a0, parameters)\n",
        "print(\"a[4][1] = \", a[4][1])\n",
        "print(\"a.shape = \", a.shape)\n",
        "print(\"y_pred[1][3] =\", y_pred[1][3])\n",
        "print(\"y_pred.shape = \", y_pred.shape)\n",
        "print(\"caches[1][1][3] =\", caches[1][1][3])\n",
        "print(\"len(caches) = \", len(caches))"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "a[4][1] =  [-0.99999375  0.77911235 -0.99861469 -0.99833267]\n",
            "a.shape =  (5, 10, 4)\n",
            "y_pred[1][3] = [0.79560373 0.86224861 0.11118257 0.81515947]\n",
            "y_pred.shape =  (2, 10, 4)\n",
            "caches[1][1][3] = [-1.1425182  -0.34934272 -0.20889423  0.58662319]\n",
            "len(caches) =  2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_itXiKn2WBW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "68e9f6f6-c9f5-4f56-ff07-acc5fec9e734"
      },
      "source": [
        ""
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.61803685, -2.03720123, -1.94258918],\n",
              "       [-2.50644065, -2.11416392, -0.41163916]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XeEMbDNY3sGQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}