Linear_Regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Linear_Regression",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyOi8q8YaGja54l6oPl74Q2E",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayushm-agrawal/ml-algorithms-numpy/blob/master/Linear_Regression.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mp-zHCVVe5UH",
        "colab_type": "text"
      },
      "source": [
        "###Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzCCGzxcevaj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import sklearn\n",
        "import numpy as np\n",
        "np.seterr(divide='ignore', invalid='ignore')\n",
        "import seaborn as sns\n",
        "import statsmodels.api as sm\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.preprocessing import normalize\n",
        "from sklearn.model_selection import train_test_split\n",
        "# from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score, roc_curve,classification_report"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RDxs26gNfr_C",
        "colab_type": "text"
      },
      "source": [
        "### Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fVlpWOvfPOg",
        "colab_type": "code",
        "outputId": "906bff0b-67e5-4429-9586-bef3b02304f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "root = 'gdrive/My Drive/DeepLearning/MNIST'"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ue2yhiA7gbLT",
        "colab_type": "text"
      },
      "source": [
        "#### Print GPU Info"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZcUDEYTfu-D",
        "colab_type": "code",
        "outputId": "cafc1035-2838-4b55-8c3d-1313fcef758e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Select the Runtime → \"Change runtime type\" menu to enable a GPU accelerator, ')\n",
        "  print('and then re-execute this cell.')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thu Feb 20 23:00:41 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.48.02    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    28W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLKz9K1YgdJz",
        "colab_type": "code",
        "outputId": "bbc5fe70-d2ac-4961-dc5f-242da1e15d1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66
        }
      },
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('To enable a high-RAM runtime, select the Runtime → \"Change runtime type\"')\n",
        "  print('menu, and then select High-RAM in the Runtime shape dropdown. Then, ')\n",
        "  print('re-execute this cell.')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Your runtime has 27.4 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0WKu_rgiXPp",
        "colab_type": "text"
      },
      "source": [
        "### Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mfv2OWAJgx6a",
        "colab_type": "code",
        "outputId": "f177df31-9e65-4a14-f1e2-f1b49b05ce2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "train_data = np.load('/content/gdrive/My Drive/DeepLearning/MNIST/mnist_train_images.npy')\n",
        "train_labels = np.load('/content/gdrive/My Drive/DeepLearning/MNIST/mnist_train_labels.npy')\n",
        "\n",
        "test_data = np.load('/content/gdrive/My Drive/DeepLearning/MNIST/mnist_test_images.npy')\n",
        "test_labels = np.load('/content/gdrive/My Drive/DeepLearning/MNIST/mnist_test_labels.npy')\n",
        "print(\"Train Data: {}... \\tTrain Labels: {}...\".format(train_data.shape, train_labels.shape))\n",
        "print(\"Test Data: {}... \\tTest Labels: {}...\".format(test_data.shape, test_labels.shape))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Data: (55000, 784)... \tTrain Labels: (55000, 10)...\n",
            "Test Data: (10000, 784)... \tTest Labels: (10000, 10)...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sL3VzQwQIiEb",
        "colab_type": "text"
      },
      "source": [
        "### Metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cs-3uhmhIjw4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Metrics:\n",
        "\n",
        "  def __init__(self, X, y):\n",
        "    self.X = X\n",
        "    self.y = y\n",
        "    self.n_samples = len(y)\n",
        "\n",
        "  def cost(self, params):\n",
        "    '''\n",
        "      Half squared mean error\n",
        "      Returns:\n",
        "        cost of the current step\n",
        "    '''\n",
        "    hypothesis = self.X @ params\n",
        "    cost = 1/(2*self.n_samples)* np.sum(hypothesis - self.y)**2\n",
        "\n",
        "    return cost\n",
        "  \n",
        "  def gradient_descent(self, params, learn_rate, n_iters):\n",
        "    ''' \n",
        "    Calculates the gradient descent algorithm\n",
        "    to minimize the cost\n",
        "    Args:\n",
        "      learn_rate - how big steps to take while minimizing the cost\n",
        "      n_iters  - number of iterations for gradient descent\n",
        "    Returns:\n",
        "      cost_history - numpy ndarray of updated cost value after each iteration\n",
        "      params - updated parameters after minimizing the cost\n",
        "    '''\n",
        "    cost_history = np.zeros((n_iters, 1))\n",
        "    print(\"Number of Iterations: {}\".format(n_iters))\n",
        "    for i in range(n_iters):\n",
        "      print(\"Current iter: {}\".format(i+1), end=\"\\r\")\n",
        "      params = params - (learn_rate/self.n_samples) * self.X.T @ (self.X @ params - self.y)\n",
        "      cost_history[i] = self.cost(params)\n",
        "    print(\"\")\n",
        "    return (cost_history, params)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RAZAqiE8QqsZ",
        "colab_type": "text"
      },
      "source": [
        "### Linear Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7huAkku6Qspx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyLinearRegression:\n",
        "  def __init__(self, X, y, learn_rate=0.03, n_iters=1500):\n",
        "\n",
        "    self.learn_rate = learn_rate\n",
        "    self.n_iter = n_iters\n",
        "    self.n_samples = len(y)\n",
        "    # self.X = np.hstack((np.ones(\n",
        "    #         (self.n_samples, 1)), (X - np.mean(X, 0)) / np.std(X, 0)))\n",
        "    # self.y = y[:, np.newaxis]\n",
        "    self.X = np.hstack((np.ones((self.n_samples, 1)), X))\n",
        "    self.y = y\n",
        "    self.n_features = np.size(X, 1)\n",
        "    self.classes = np.size(y, 1)\n",
        "    # self.params = np.zeros((self.n_features + 1, 1))\n",
        "    # self.coef_ = None\n",
        "    # self.intercept_ = None\n",
        "\n",
        "  def fit(self):\n",
        "    print(\"Number of Iterations: {}\".format(self.n_iter))\n",
        "    print(\"Features.shape: {}\".format(self.X.shape))\n",
        "    print(\"Labels shape: {}\".format(self.y.shape))\n",
        "    print(\"Number of features: {}\".format(self.n_features))\n",
        "    params = np.zeros((self.n_features + 1, self.classes))\n",
        "    print(\"Params shape: {}\".format(params.shape))\n",
        "    for i in range(self.n_iter):\n",
        "      print(\"Current iteration: {}\".format(i+1), end=\"\\r\")\n",
        "      params = params - (self.learn_rate/self.n_samples) * self.X.T @ (self.X @ params - self.y)\n",
        "      # print(\"Params: {}\".format(self.params), end=\"\\r\")\n",
        "    # self.intercept_ = self.params[0]\n",
        "    # self.coef_ = self.params[1:]\n",
        "    \n",
        "    return params\n",
        "\n",
        "  def score(self, params, X=None, y=None):\n",
        "\n",
        "    if X is None:\n",
        "      X = self.X\n",
        "    else:\n",
        "      n_samples = np.size(X,0)\n",
        "      X = np.hstack((np.ones((n_samples, 1)), X))\n",
        "    \n",
        "    if y is None:\n",
        "      y = self.y\n",
        "    # else:\n",
        "    #   y = y[:, np.newaxis]\n",
        "\n",
        "    y_pred = X @ params\n",
        "    score = 1 - (((y-y_pred)**2).sum() / ((y-y.mean())**2).sum())\n",
        "\n",
        "    return score\n",
        "\n",
        "  def predict(self, params):\n",
        "    n_samples = np.size(self.X, 0)\n",
        "    y = self.X @ params\n",
        "    return y\n",
        "\n",
        "  def get_params(self):\n",
        "\n",
        "    return self.params"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kQNN7hRhP6X5",
        "colab_type": "text"
      },
      "source": [
        "### Normalize Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ihz67TXgP1Yh",
        "colab_type": "code",
        "outputId": "9c82e7c0-479d-4098-fc27-f146f4d50c22",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "# utilizing SK-Learn to normalize the data before passing it through the gradient descent function\n",
        "X_sk = normalize(train_data, norm='l2', axis=1, copy=True, return_norm=False)\n",
        "y_sk = train_labels\n",
        "print(X_sk.shape)\n",
        "print(y_sk.shape)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(55000, 784)\n",
            "(55000, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "brnCzjFjXmKv",
        "colab_type": "text"
      },
      "source": [
        "### Running Gradient Descent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8L1LdWwXrla",
        "colab_type": "code",
        "outputId": "5aaafac6-f8ad-4c5e-b68b-905d029179f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "n_iters = 1500\n",
        "learning_rate = 0.01\n",
        "X_sk = np.hstack((np.ones((len(y_sk),1)), X_sk))\n",
        "\n",
        "params = np.zeros((np.size(X_sk, 1), 1))\n",
        "metrics = Metrics(X_sk, y_sk)\n",
        "init_cost = metrics.cost(params)\n",
        "\n",
        "print(\"Initial Cost: {}\".format(init_cost))\n",
        "\n",
        "(cost_history, params) = metrics.gradient_descent(params, learning_rate, n_iters)\n",
        "\n",
        "print(\"Optimal parameters are: \\n\", params, \"\\n\")\n",
        "\n",
        "print(\"Final cost is: \", cost_history[-1])\n",
        "\n",
        "plt.plot(range(len(cost_history)), cost_history, 'r')\n",
        "\n",
        "plt.title(\"Convergence Graph of Cost Function\")\n",
        "plt.xlabel(\"Number of Iterations\")\n",
        "plt.ylabel(\"Cost\")\n",
        "plt.show()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Initial Cost: 27500.0\n",
            "Number of Iterations: 1500\n",
            "\n",
            "Optimal parameters are: \n",
            " [[0.07917581 0.1191234  0.07171404 ... 0.09211563 0.02821782 0.05442129]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " ...\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]] \n",
            "\n",
            "Final cost is:  [0.04549526]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEWCAYAAACnlKo3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3debxcdX3G8c9DNrKRhCSGkAQSIKBI\nIWCEUKlFRAi0GhTZSiVYBBdQqbYatC0Ul6JU6wKiKAhUBCJLSS2KgNFqK4GwJWyBK1sSEhIIhAAF\nEvj2j99vLic3d53MzLk387xfr/Oamd/ZvufcO/PMWeYcRQRmZmbV2KrsAszMrO9yiJiZWdUcImZm\nVjWHiJmZVc0hYmZmVXOImJlZ1RwiZr2MpBMl/b5G0xos6T8lrZX0s1pMc0si6fuS/rHsOvoyh8gW\nStJfSVoo6QVJKyT9QtIBZdfVF0l6j6T5ktZJekbS3ZI+L2nrsmvrhg8C44DREXFUewNI2lXSzyQ9\nncNmkaTPSOpX7UwlnSXpJ10M85ik/8v/o5Vu+2rn2Y2aNgnniPhYRHypXvNsBg6RLZCkzwDfAr5K\n+gDZAfgeMKvMuook9S+7hu6QdBRwNfBTYMeIGA0cA0wEJnUwTm9ath2BhyJiQ3s9Je0MLACWAn8S\nESOAo4DpwPAG1PfeiBhW6J5swDytliLC3RbUASOAF4CjOhlmEClknszdt4BBud+BwDLgs8AqYAXw\n4dxvP2Al0K8wrfcDi/LzrYA5wB+BZ4C5wLa532QggJOAJ4D/zu0nAI/n4f8ReAw4uAfTm52n9zTw\nxUJd/YAv5HHXAXcAk3K/NwM3AWuAJcDRHawnkT5cP9vFOj+LFDQ/AZ4HPgLsC/wBeC6vw/OAgYVx\nAvgU8Eiu/Vxgq9zvROD3wL8CzwKPAod1Mv+3AL/J87oPeF9u/2fgVWB9/p84qZ1xfwL8VxfL9748\n3efyfN5S6Pd5YHlex0uAdwMz28z3ng6m2/q3btN+ILCso2Hz+p4LXJbnex8wvTDsJOBaYHX+vzkv\nr6OXgddyTc/lYS8BvlwY92SgJf9vzAO2b/M3+xjwcF4X5wMq+z1fdld6Ae5q/AdNb+ANQP9Ohjkb\nuBV4EzAW+F/gS7nfgXn8s4EBwOHAS8Co3P+PwHsK0/oZMCc//3Se7kRSUP0AuCL3m5zfhJcBQ4HB\nwO75DX0AMJD0obm+8GHRnen9ME9rL+CVygcc8PfAYmA3UhjsBYzO814KfBjoD+xN+hDfvZ319OY8\nj8ldrPOzct1HkIJvMPA2YEaex2TgAeD0wjgBzAe2JW0pPgR8JPc7MU/vZFIYfpwU9pt8YOW/UQsp\nMAcCB5E+WHcr1PaTTmpfSf6S0EH/XYEXgffkeX0uz29gXrdLyR+0eTl37s588zCPUX2IvEz63+wH\n/Atwa+7XD7gH+Lf8t94aOKCwXn/fZrqXkEMkr7ungX1I/2/fJX/ZKfzNfg6MzH+z1cDMst/zZXel\nF+Cuxn9QOB5Y2cUwfwQOL7w+FHgsPz8Q+D8KIUTaIpmRn38ZuDg/H54/YHbMrx8A3l0Yb3z+MKx8\nkAawU6H/P5FDIb8eQvoGe3APpjex0P824Nj8fAkwq51lPwb4XZu2HwBntjPsAXkeWxfariR9C30J\n+FBuO6v4YdPBOj8duK7wOoofQMAngFvy8xOBljbrJYDt2pnun5GCYKtC2xXAWYXaOguR9Z19EJK2\nDucWXm9F2vI4ENgl/28cDAxoM16n883DPEbeKsjdfxT+B7sKkZsL/XYH/i8/35/04b7Jlyi6DpGL\ngK8X+g3L62dy4W92QKH/XPIXqGbufExky/MMMKaL/fLbk3YhVTye21qnERvvQ3+J9IaCdGzgA5IG\nAR8A7oyIyrR2BK6T9Jyk50gh8BrpuEzF0jZ1tL6OiJdy/RXdmd7KDuqcRArLtnYE9qtMM0/3eGC7\ndoat1DK+UOOxETESuJP0rbe95aocrP65pJWSnicdnxrTZvrFcdr+DVqXK68XCstWtD2wNCJebzOt\nCe0M255nKCxfB9Nv/V/J81kKTIiIFlI4ngWsknRlFQfGj4iIkbk7ogfjtf27b53/5ycBj0cHx4C6\n0HZZXyCtn+K67Oj/rWk5RLY8fyDt1unsDfkk6cO0Yofc1qWIuJ/0RjsM+CtSqFQsJe27H1noto6I\n5cVJFJ6vIO2qAtLpqKRdTj2ZXkeWAjt30P7bNtMcFhEfb2fYJaRv3R/oxvyizesLgAeBqRGxDWl3\nk9oMUzww3+2/QRtPApMkFd/LO5Dq7o6bgSO7mH7r/4okkepeDhARP42IA/IwAXwtD9p2ffTEi6St\nr8o8+5F2u3bHUmCHDr5EdVVT22UdSvp/7O66bEoOkS1MRKwl7SY6X9IRkoZIGiDpMElfz4NdAfyD\npLGSxuThOz0ds42fko5XvJN0TKTi+8BXJO0IkKff2RlhVwPvlfSnkgaSvtEWP2h7Or2iHwFfkjRV\nyZ6SRpP2ae8q6UN5vQyQ9HZJb2k7gfyt+7PAmZJOljQqT2sqG28NtWc46SD7C5LeTDqu0dbf52lO\nIq3Pq7q5bEULSN+IP5eX5UDgvaTdbt1xJvCnks6VtB2ApF0k/UTSSNIum7+Q9G5JA0jr4xXgfyXt\nJumgvFX6Mmk3aGWL6Clgcptw666HSFsWf5Hn+Q+kYxTdcRvpy8k5koZK2lrSOwo1Tcz/a+25Aviw\npGl5mb4KLIiIx6pYhqbhENkCRcQ3gM+Q3nyrSd/OTgP+Iw/yZWAhsIh08PnO3NZdVwB/Dvw6Ip4u\ntH+bdEbLryStIx0U36+TOu8DPkn6wFtB2j++ivQh1ePptfFN0gfgr0gf5hcBgyNiHXAIcCzpm+dK\n0rfndj+kIuIq4Gjgr0nr8ek83QvZOEDb+jvSlto60sH/9gLietJZY3cD/5Vr7JGIeJUUGofl2r4H\nnBARD3Zz/D+SjiNMBu6TtBa4hvT/sS4ilpCW/bt5+u8lnZb7KmmdnZPbV5JO1DgjT7qybp6RdGcP\nl2kt6RjRj0hbAS+Szhjszriv5Rp3IZ21t4x0HAzg16QzuVZKerqdcW8mHQO6hvT/uDPp/8Q6oXyA\nyKx0koaRDrBOjYhHy66nniQFaTlbyq7FbHN4S8RKJem9eZfbUNIpvotJZ+KYWR/gELGyzeKNHz1O\nJZ2i681jsz7Cu7PMzKxq3hIxM7Oq9aYLxTXEmDFjYvLkyWWXYWbWp9xxxx1PR8Qmv9dpuhCZPHky\nCxcuLLsMM7M+RdLj7bV7d5aZmVXNIWJmZlVziJiZWdUcImZmVjWHiJmZVc0hYmZmVXOImJlZ1Rwi\n3XXeeXBVNbd7MDPbcjlEuusHP4C5c8uuwsysV3GIdNeQIfDSS10PZ2bWRBwi3TV0qEPEzKwNh0h3\nDRkCL75YdhVmZr2KQ6S7vDvLzGwTDpHucoiYmW3CIdJdDhEzs004RLrLIWJmtgmHSHdVQsT3pDcz\na+UQ6a4hQ1KAvPJK2ZWYmfUadQsRSZMkzZd0v6T7JH06t58labmku3N3eGGcMyS1SFoi6dBC+8zc\n1iJpTqF9iqQFuf0qSQPrtTwMHZoefZqvmVmrem6JbAA+GxG7AzOAUyXtnvv9W0RMy90NALnfscBb\ngZnA9yT1k9QPOB84DNgdOK4wna/lae0CPAucVLelGTIkPfq4iJlZq7qFSESsiIg78/N1wAPAhE5G\nmQVcGRGvRMSjQAuwb+5aIuKRiHgVuBKYJUnAQcDVefxLgSPqszQ4RMzM2tGQYyKSJgN7Awty02mS\nFkm6WNKo3DYBWFoYbVlu66h9NPBcRGxo097e/E+RtFDSwtWrV1e3EA4RM7NN1D1EJA0DrgFOj4jn\ngQuAnYFpwArgG/WuISIujIjpETF97Nix1U3EIWJmton+9Zy4pAGkALk8Iq4FiIinCv1/CPw8v1wO\nTCqMPjG30UH7M8BISf3z1khx+NpziJiZbaKeZ2cJuAh4ICK+WWgfXxjs/cC9+fk84FhJgyRNAaYC\ntwG3A1PzmVgDSQff50VEAPOBD+bxZwPX12t5WkPEZ2eZmbWq55bIO4APAYsl3Z3bvkA6u2oaEMBj\nwEcBIuI+SXOB+0lndp0aEa8BSDoNuBHoB1wcEffl6X0euFLSl4G7SKFVH94SMTPbRN1CJCJ+D6id\nXjd0Ms5XgK+0035De+NFxCOks7fqzyFiZrYJ/2K9uyo/NnSImJm1coh0l7dEzMw24RDprkGDQHKI\nmJkVOES6S/Ll4M3M2nCI9ITvs25mthGHSE94S8TMbCMOkZ5wiJiZbcQh0hMOETOzjThEemLoUIeI\nmVmBQ6QnvCViZrYRh0hP+OwsM7ONOER6wiFiZrYRh0hP+JiImdlGHCI9MWwYvPBC2VWYmfUaDpGe\nGDYsbYm89lrZlZiZ9QoOkZ4YNiw9epeWmRngEOmZSoh4l5aZGeAQ6RmHiJnZRhwiPeEQMTPbiEOk\nJxwiZmYbcYj0hEPEzGwjDpGecIiYmW3EIdITDhEzs404RHrCIWJmthGHSE84RMzMNuIQ6YlBg6Bf\nP4eImVnmEOkJyRdhNDMrcIj0lEPEzKxV3UJE0iRJ8yXdL+k+SZ/O7dtKuknSw/lxVG6XpO9IapG0\nSNI+hWnNzsM/LGl2of1tkhbncb4jSfVanlYOETOzVvXcEtkAfDYidgdmAKdK2h2YA9wSEVOBW/Jr\ngMOAqbk7BbgAUugAZwL7AfsCZ1aCJw9zcmG8mXVcnsQhYmbWqm4hEhErIuLO/Hwd8AAwAZgFXJoH\nuxQ4Ij+fBVwWya3ASEnjgUOBmyJiTUQ8C9wEzMz9tomIWyMigMsK06ofh4iZWauGHBORNBnYG1gA\njIuIFbnXSmBcfj4BWFoYbVlu66x9WTvt7c3/FEkLJS1cvXr1Zi2LQ8TM7A11DxFJw4BrgNMj4vli\nv7wFEfWuISIujIjpETF97Nixmzcxh4iZWau6hoikAaQAuTwirs3NT+VdUeTHVbl9OTCpMPrE3NZZ\n+8R22uvLIWJm1qqeZ2cJuAh4ICK+Weg1D6icYTUbuL7QfkI+S2sGsDbv9roROETSqHxA/RDgxtzv\neUkz8rxOKEyrfhwiZmat+tdx2u8APgQslnR3bvsCcA4wV9JJwOPA0bnfDcDhQAvwEvBhgIhYI+lL\nwO15uLMjYk1+/gngEmAw8Ivc1VclRCLSjw/NzJpY3UIkIn4PdPQp++52hg/g1A6mdTFwcTvtC4E9\nNqPMnhs2DF5/HV5+GQYPbuiszcx6G/9ivacqF2Fct67cOszMegGHSE8NH54efVzEzMwh0mPbbJMe\nn3++8+HMzJqAQ6SnKiGydm25dZiZ9QIOkZ4aMSI9ekvEzMwh0mPenWVm1soh0lMOETOzVg6RnnKI\nmJm1coj01ODB0L+/D6ybmeEQ6TkpbY14S8TMzCFSFYeImRngEKmOQ8TMDHCIVMchYmYGOESqM2KE\nD6ybmeEQqY63RMzMAIdIdRwiZmaAQ6Q6DhEzM8AhUp0RI9KdDV99texKzMxK5RCpRuXSJ767oZk1\nOYdINXxPETMzwCFSHV+E0cwMcIhUxyFiZgY4RKpTubuhd2eZWZNziFTDIWJmBjhEqjNqVHp89tly\n6zAzK5lDpBojR6ZHh4iZNTmHSDX694dhwxwiZtb0HCLVGjXKIWJmTa9uISLpYkmrJN1baDtL0nJJ\nd+fu8EK/MyS1SFoi6dBC+8zc1iJpTqF9iqQFuf0qSQPrtSztcoiYmdV1S+QSYGY77f8WEdNydwOA\npN2BY4G35nG+J6mfpH7A+cBhwO7AcXlYgK/lae0CPAucVMdl2dSoUfDccw2dpZlZb1O3EImI/wbW\ndHPwWcCVEfFKRDwKtAD75q4lIh6JiFeBK4FZkgQcBFydx78UOKKmC9AVb4mYmZVyTOQ0SYvy7q58\nriwTgKWFYZblto7aRwPPRcSGNu3tknSKpIWSFq5evbo2S+EQMTNreIhcAOwMTANWAN9oxEwj4sKI\nmB4R08eOHVubiY4c6RAxs6bX0BCJiKci4rWIeB34IWl3FcByYFJh0Im5raP2Z4CRkvq3aW+cUaPg\nxRdh/fqGztbMrDdpaIhIGl94+X6gcubWPOBYSYMkTQGmArcBtwNT85lYA0kH3+dFRADzgQ/m8WcD\n1zdiGVr5V+tmZt0LEUn/3p22Nv2vAP4A7CZpmaSTgK9LWixpEfAu4G8BIuI+YC5wP/BL4NS8xbIB\nOA24EXgAmJuHBfg88BlJLaRjJBd1Z1lqxiFiZkb/rgcB0qm3rfKpt2/rbISIOK6d5g4/6CPiK8BX\n2mm/AbihnfZHeGN3WONVQsSn+ZpZE+t0SyT/AHAdsKek53O3DlhFo3cf9TbeEjEz6zxEIuJfImI4\ncG5EbJO74RExOiLOaFCNvZNDxMys2wfWfy5pKICkv5b0TUk71rGu3s9X8jUz63aIXAC8JGkv4LPA\nH4HL6lZVX+AtETOzbofIhnxa7SzgvIg4Hxhev7L6gEGDYMgQWNPdK7uYmW15unt21jpJZwAfAv5M\n0lbAgPqV1UeMHg3PPFN2FWZmpenulsgxwCvA30TEStIvxM+tW1V9xZgx8PTTZVdhZlaaboVIDo7L\ngRGS/hJ4OSKa+5gIOETMrOl19xfrR5MuQ3IUcDSwQNIHOx+rCThEzKzJdfeYyBeBt0fEKgBJY4Gb\neeN+Hs3JIWJmTa67x0S2qgRI9kwPxt1yjRkDa9f6Sr5m1rS6uyXyS0k3Alfk18fQzvWsms6YMelx\nzRoYN67cWszMStBpiEjaBRgXEX8v6QPAAbnXH0gH2ptbJUSeftohYmZNqastkW8BZwBExLXAtQCS\n/iT3e29dq+vtiiFiZtaEujquMS4iFrdtzG2T61JRX+IQMbMm11WIjOyk3+BaFtInOUTMrMl1FSIL\nJZ3ctlHSR4A76lNSHzJ6dHp0iJhZk+rqmMjpwHWSjueN0JgODCTdI725DRoEw4c7RMysaXUaIhHx\nFPCnkt4F7JGb/ysifl33yvoK/+DQzJpYt34nEhHzgfl1rqVvcoiYWRPzr84319ix8NRTZVdhZlYK\nh8jmGjfOIWJmTcshsrm22w5WrYLXXy+7EjOzhnOIbK7ttoMNG3ybXDNrSg6RzVW5ZpZ3aZlZE3KI\nbK7ttkuPK1eWW4eZWQkcIpursiXiEDGzJuQQ2VyVLRHvzjKzJlS3EJF0saRVku4ttG0r6SZJD+fH\nUbldkr4jqUXSIkn7FMaZnYd/WNLsQvvbJC3O43xHkuq1LJ0aMSJd/sRbImbWhOq5JXIJMLNN2xzg\nloiYCtySXwMcBkzN3SnABZBCBzgT2A/YFzizEjx5mJML47WdV2NI/q2ImTWtuoVIRPw30Pa811nA\npfn5pcARhfbLIrkVGClpPHAocFNErImIZ4GbgJm53zYRcWtEBHBZYVqNt9123hIxs6bU6GMi4yJi\nRX6+EqjcU3YCsLQw3LLc1ln7snba2yXpFEkLJS1cvXr15i1BexwiZtakSjuwnrcgokHzujAipkfE\n9LFjx9Z+Bt6dZWZNqtEh8lTeFUV+XJXblwOTCsNNzG2dtU9sp70c48enS5+sX19aCWZmZWh0iMwD\nKmdYzQauL7SfkM/SmgGszbu9bgQOkTQqH1A/BLgx93te0ox8VtYJhWk13sSJEOFdWmbWdLp1P5Fq\nSLoCOBAYI2kZ6Syrc4C5kk4CHgeOzoPfABwOtAAvAR8GiIg1kr4E3J6HOzsiKgfrP0E6A2ww8Ivc\nlWNi3ihatgwmTep8WDOzLUjdQiQijuug17vbGTaAUzuYzsXAxe20L+SNuy2WqxgiZmZNxL9YrwWH\niJk1KYdILYwcCUOGOETMrOk4RGpBSlsjDhEzazIOkVpxiJhZE3KI1IpDxMyakEOkViZOhCefhNde\nK7sSM7OGcYjUysSJ6V7rq1Z1PayZ2RbCIVIrlR8ZPvFEuXWYmTWQQ6RWJk9Oj48+WmoZZmaN5BCp\nFYeImTUhh0itDBsGY8c6RMysqThEamnKFIeImTUVh0gtOUTMrMk4RGppypR0dpZ/K2JmTcIhUks7\n7ZTubri8vJssmpk1kkOklqZMSY/epWVmTcIhUkuVEHnkkXLrMDNrEIdILe2wAwwYAA8/XHYlZmYN\n4RCppQEDYOed4cEHy67EzKwhHCK1tttusGRJ2VWYmTWEQ6TW3vxmaGnxab5m1hQcIrW2227w6qvw\n2GNlV2JmVncOkVrbbbf06OMiZtYEHCK1VgkRHxcxsybgEKm10aNhzBhviZhZU3CI1MMee8DixWVX\nYWZWdw6RethzzxQir79ediVmZnXlEKmHvfaCF1/05U/MbItXSohIekzSYkl3S1qY27aVdJOkh/Pj\nqNwuSd+R1CJpkaR9CtOZnYd/WNLsMpalXXvumR4XLSq3DjOzOitzS+RdETEtIqbn13OAWyJiKnBL\nfg1wGDA1d6cAF0AKHeBMYD9gX+DMSvCU7q1vha22gnvuKbsSM7O66k27s2YBl+bnlwJHFNovi+RW\nYKSk8cChwE0RsSYingVuAmY2uuh2DR4Mu+7qLREz2+KVFSIB/ErSHZJOyW3jImJFfr4SGJefTwCW\nFsZdlts6at+EpFMkLZS0cPXq1bVahs7ttRfcdVdj5mVmVpKyQuSAiNiHtKvqVEnvLPaMiCAFTU1E\nxIURMT0ipo8dO7ZWk+3cvvvC44/DypWNmZ+ZWQlKCZGIWJ4fVwHXkY5pPJV3U5EfV+XBlwOTCqNP\nzG0dtfcOM2akxwULyq3DzKyOGh4ikoZKGl55DhwC3AvMAypnWM0Grs/P5wEn5LO0ZgBr826vG4FD\nJI3KB9QPyW29w957p/uL3Hpr2ZWYmdVN/xLmOQ64TlJl/j+NiF9Kuh2YK+kk4HHg6Dz8DcDhQAvw\nEvBhgIhYI+lLwO15uLMjYk3jFqMLgwfDtGneEjGzLVrDQyQiHgH2aqf9GeDd7bQHcGoH07oYuLjW\nNdbMfvvBj3+c7i3Sr1/Z1ZiZ1VxvOsV3yzNjRvrl+r33ll2JmVldOETq6YAD0uNvf1tuHWZmdeIQ\nqacdd4RddoGbby67EjOzunCI1NvBB8NvfgPr15ddiZlZzTlE6u3gg2HdOrjttrIrMTOrOYdIvb3r\nXSB5l5aZbZEcIvW27bYwfTrccEPZlZiZ1ZxDpBHe//60O2vp0q6HNTPrQxwijXDkkenxuuvKrcPM\nrMYcIo2w666wxx5wzTVlV2JmVlMOkUY58kj43e9gxYquhzUz6yMcIo1y/PEQAZdcUnYlZmY14xBp\nlKlT4cAD4Uc/gtdfL7saM7OacIg00sknwyOPwPz5ZVdiZlYTDpFG+sAHYPRo+Pa3y67EzKwmHCKN\ntPXW8KlPwX/+JyxaVHY1ZmabzSHSaJ/8JAwfDl/9atmVmJltNodIo40aBaedBnPnwsKFZVdjZrZZ\nHCJlmDMH3vSmtGvLZ2qZWR/mECnDNtvAOefAH/4AF11UdjVmZlVziJTlhBPgoIPg9NPhoYfKrsbM\nrCoOkbJstRVceikMGgRHHw0vvFB2RWZmPeYQKdPEiXD55XDvvXDMMbBhQ9kVmZn1iEOkbIcdBuef\nn25adeSR8PLLZVdkZtZtDpHe4KMfhfPOg3nz4D3vgSefLLsiM7NucYj0FqeeCldcAXfeCdOmwdVX\np6v+mpn1Yg6R3uTYY9MPELffHo46Km2V/M//OEzMrNdyiPQ2b3lLCpLzzoO77oIDDoD994fvfx9W\nrSq7OjOzjfT5EJE0U9ISSS2S5pRdT0307592bz3xRDrovnYtfPzjMH48zJgBn/tcOn7y6KP+xbuZ\nlUrRh3eVSOoHPAS8B1gG3A4cFxH3dzTO9OnTY2Ffu2ZVBCxenI6T/PrXcPvt8Oqrqd/gweke7pMm\npd1g22+fwmbEiPTL+Eo3fHj6TcrAgW90Awak36uYmXVB0h0RMb1te/8yiqmhfYGWiHgEQNKVwCyg\nwxDpkyTYc8/UnX12Og34rrvg/vtT9+CDsGwZLFgAq1f3bNr9+78RKFIKlWoeq12uRozTyHlVW59Z\nI9x5Z/oyWUN9PUQmAEsLr5cB+7UdSNIpwCkAO+ywQ2Mqq6ett07HSfbff9N+r76agmTtWnj++Te6\ndetSv2L3yitvPF+/Pm3xRKRdZN19rHZ3WjVbwNVuNTdqXn14q96aRB2+5PT1EOmWiLgQuBDS7qyS\ny6mvgQNhwoTUmZnVWV/fIb4cmFR4PTG3mZlZA/T1ELkdmCppiqSBwLHAvJJrMjNrGn16d1ZEbJB0\nGnAj0A+4OCLuK7ksM7Om0adDBCAibgBuKLsOM7Nm1Nd3Z5mZWYkcImZmVjWHiJmZVc0hYmZmVevT\n186qhqTVwONVjj4GeLqG5dRDb6+xt9cHvb/G3l4fuMZa6G317RgRY9s2Nl2IbA5JC9u7AFlv0ttr\n7O31Qe+vsbfXB66xFnp7fRXenWVmZlVziJiZWdUcIj1zYdkFdENvr7G31we9v8beXh+4xlro7fUB\nPiZiZmabwVsiZmZWNYeImZlVzSHSDZJmSloiqUXSnBLrmCRpvqT7Jd0n6dO5fVtJN0l6OD+Oyu2S\n9J1c9yJJ+zSozn6S7pL08/x6iqQFuY6r8mX7kTQov27J/Sc3qL6Rkq6W9KCkByTt3wvX4d/mv/G9\nkq6QtHXZ61HSxZJWSbq30Nbj9SZpdh7+YUmz61zfufnvvEjSdZJGFvqdketbIunQQnvd3u/t1Vjo\n91lJIWlMft3wdViViHDXSUe6xPwfgZ2AgcA9wO4l1TIe2Cc/Hw48BOwOfB2Yk9vnAF/Lzw8HfgEI\nmAEsaFCdnwF+Cvw8v54LHJuffx/4eH7+CeD7+fmxwFUNqu9S4CP5+UBgZG9ah6TbPj8KDC6svxPL\nXo/AO4F9gHsLbT1ab8C2wCP5cVR+PqqO9R0C9M/Pv1aob/f8Xh4ETMnv8X71fr+3V2Nun0S6pcXj\nwJiy1mFVy1TWjPtKB+wP3Fh4fQZwRtl15VquB94DLAHG57bxwJL8/AfAcYXhW4erY00TgVuAg4Cf\n5zfA04U3cuv6zG+a/fPz/nk41bm+EfkDWm3ae9M6nAAszR8S/fN6PLQ3rEdgcpsP6R6tN+A44AeF\n9o2Gq3V9bfq9H7g8P9/offIp3R4AAAbZSURBVFxZh414v7dXI3A1sBfwGG+ESCnrsKedd2d1rfKG\nrliW20qVd1nsDSwAxkXEitxrJTAuPy+j9m8BnwNez69HA89FxIZ2amitL/dfm4evpynAauDHeZfb\njyQNpRetw4hYDvwr8ASwgrRe7qB3rceKnq63Mt9Pf0P6Zk8ndTS8PkmzgOURcU+bXr2mxs44RPog\nScOAa4DTI+L5Yr9IX01KOW9b0l8CqyLijjLm3039SbsTLoiIvYEXSbthWpW5DgHycYVZpMDbHhgK\nzCyrnu4qe711RtIXgQ3A5WXXUiRpCPAF4J/KrqVaDpGuLSftr6yYmNtKIWkAKUAuj4hrc/NTksbn\n/uOBVbm90bW/A3ifpMeAK0m7tL4NjJRUuYtmsYbW+nL/EcAzdawP0re2ZRGxIL++mhQqvWUdAhwM\nPBoRqyNiPXAtad32pvVY0dP11vD1KelE4C+B43PQ9ab6diZ9Wbgnv28mAndK2q4X1dgph0jXbgem\n5jNjBpIOXM4roxBJAi4CHoiIbxZ6zQMqZ2jMJh0rqbSfkM/ymAGsLex6qLmIOCMiJkbEZNJ6+nVE\nHA/MBz7YQX2Vuj+Yh6/rN9mIWAkslbRbbno3cD+9ZB1mTwAzJA3Jf/NKjb1mPRb0dL3dCBwiaVTe\n4jokt9WFpJmk3avvi4iX2tR9bD6zbQowFbiNBr/fI2JxRLwpIibn980y0skzK+kl67BLZR2M6Usd\n6SyJh0hnbXyxxDoOIO0uWATcnbvDSfu/bwEeBm4Gts3DCzg/170YmN7AWg/kjbOzdiK9QVuAnwGD\ncvvW+XVL7r9Tg2qbBizM6/E/SGe49Kp1CPwz8CBwL/DvpLOISl2PwBWkYzTrSR92J1Wz3kjHJlpy\n9+E619dCOn5Qeb98vzD8F3N9S4DDCu11e7+3V2Ob/o/xxoH1hq/Dajpf9sTMzKrm3VlmZlY1h4iZ\nmVXNIWJmZlVziJiZWdUcImZmVjWHiPVp+aqn3yi8/jtJZ9Vo2pdI+mDXQ272fI5Suprw/DbtkytX\ne5U0TdLhNZznSEmfKLzeXtLVtZq+NQ+HiPV1rwAfqFw+u7co/LK8O04CTo6Id3UyzDTS7xdqVcNI\n0tV/AYiIJyOi7oFpWx6HiPV1G0j3ov7btj3abklIeiE/Hijpt5Kul/SIpHMkHS/pNkmLJe1cmMzB\nkhZKeihfG6xyv5RzJd2e7/Pw0cJ0fydpHukX5m3rOS5P/15JX8tt/0T6EelFks5tbwHzL6fPBo6R\ndLekYyQNVbo3xW35QpKz8rAnSpon6dfALZKGSbpF0p153rPyZM8Bds7TO7fNVs/Wkn6ch79L0rsK\n075W0i+V7mPx9cL6uCQv12JJm/wtbMvVk29LZr3V+cCiyodaN+0FvAVYQ7ofw48iYl+lG319Ejg9\nDzcZ2Jd0jaP5knYBTiBdguLtkgYB/yPpV3n4fYA9IuLR4swkbU+6n8XbgGeBX0k6IiLOlnQQ8HcR\nsbC9QiPi1Rw20yPitDy9r5Iub/I3Sjdauk3SzYUa9oyINXlr5P0R8XzeWrs1h9ycXOe0PL3JhVme\nmmYbfyLpzbnWXXO/aaSrR78CLJH0XeBNwISI2CNPayTWNLwlYn1epCsZXwZ8qgej3R4RKyLiFdJl\nJSohsJgUHBVzI+L1iHiYFDZvJl2r6ARJd5MuxT+adO0lgNvaBkj2duA3kS6qWLma7Dt7UG9bhwBz\ncg2/IV36ZIfc76aIWJOfC/iqpEWky5JM4I3LtXfkAOAnABHxIOlGSZUQuSUi1kbEy6StrR1J62Un\nSd/N16p6vp1p2hbKWyK2pfgWcCfw40LbBvIXJUlbke5UV/FK4fnrhdevs/H7ou11gYL0wfzJiNjo\noneSDiRdWr4RBBwZEUva1LBfmxqOB8YCb4uI9UpXit16M+ZbXG+vkW6S9aykvUg3zvoYcDTp2k7W\nBLwlYluE/M17LukgdcVjpN1HAO8DBlQx6aMkbZWPk+xEuljfjcDHlS7Lj6RdlW5s1ZnbgD+XNEZS\nP9Ld6X7bgzrWkW6JXHEj8ElJyjXs3cF4I0j3eFmfj23s2MH0in5HCh/ybqwdSMvdrrybbKuIuAb4\nB9LuNGsSDhHbknwDKJ6l9UPSB/c9pNueVrOV8AQpAH4BfCzvxvkRaVfOnflg9A/oYqs+0iW855Au\n534PcEdEXN/ZOG3MB3avHFgHvkQKxUWS7suv23M5MF3SYtKxnAdzPc+QjuXc284B/e8BW+VxrgJO\nzLv9OjIB+E3etfYT0i1lrUn4Kr5mZlY1b4mYmVnVHCJmZlY1h4iZmVXNIWJmZlVziJiZWdUcImZm\nVjWHiJmZVe3/AXCR7e+qGOZTAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cQcHQGFAZ4HI",
        "colab_type": "text"
      },
      "source": [
        "### Comparing with SK-Learn Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsOqGk7yYqJq",
        "colab_type": "code",
        "outputId": "ac354719-115c-4dcb-ca52-45b73002766b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "\n",
        "X_sk = normalize(train_data, norm='l2', axis=1, copy=True, return_norm=False)\n",
        "y_sk = train_labels\n",
        "lr = MyLinearRegression(X_sk, y_sk, learn_rate=0.03, n_iters=1500)\n",
        "optimal_params = lr.fit()\n",
        "print(\"Optimal Params: \\n{}\\n\".format(optimal_params))\n",
        "sk_learn_lr = LinearRegression().fit(X_sk, y_sk)\n",
        "\n",
        "accuracy_train = lr.score(optimal_params)\n",
        "sk_accuracy_train = sk_learn_lr.score(X_sk, y_sk)\n",
        "print(\"Train accuracy: {}\\n\".format(accuracy_train))\n",
        "\n",
        "X_test = normalize(test_data, norm='l2', axis=1, copy=True, return_norm=False)\n",
        "y_test = test_labels\n",
        "\n",
        "accuracy_test = lr.score(optimal_params, X = X_test, y = y_test)\n",
        "print(\"Test Acccuracy: {}\\n\".format(accuracy_test))\n",
        "sk_accuracy_test = sk_learn_lr.score(X_test, y_test)\n",
        "print(\"\\n\\n\")\n",
        "pd.DataFrame([[accuracy_train, sk_accuracy_train],\n",
        "              [accuracy_test, sk_accuracy_test]],\n",
        "             ['Training Accuracy', 'Test Accuracy'],    \n",
        "             ['Our Implementation', 'Sklearn\\'s Implementation'])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of Iterations: 1500\n",
            "Features.shape: (55000, 785)\n",
            "Labels shape: (55000, 10)\n",
            "Number of features: 784\n",
            "Params shape: (785, 10)\n",
            "Optimal Params: \n",
            "[[0.07917581 0.1191234  0.07171404 ... 0.09211563 0.02821782 0.05442129]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " ...\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]]\n",
            "\n",
            "Train accuracy: 0.26072651895122234\n",
            "\n",
            "Test Acccuracy: 0.2643806455082093\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:434: FutureWarning: The default value of multioutput (not exposed in score method) will change from 'variance_weighted' to 'uniform_average' in 0.23 to keep consistent with 'metrics.r2_score'. To specify the default value manually and avoid the warning, please either call 'metrics.r2_score' directly or make a custom scorer with 'metrics.make_scorer' (the built-in scorer 'r2' uses multioutput='uniform_average').\n",
            "  \"multioutput='uniform_average').\", FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:434: FutureWarning: The default value of multioutput (not exposed in score method) will change from 'variance_weighted' to 'uniform_average' in 0.23 to keep consistent with 'metrics.r2_score'. To specify the default value manually and avoid the warning, please either call 'metrics.r2_score' directly or make a custom scorer with 'metrics.make_scorer' (the built-in scorer 'r2' uses multioutput='uniform_average').\n",
            "  \"multioutput='uniform_average').\", FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Our Implementation</th>\n",
              "      <th>Sklearn's Implementation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Training Accuracy</th>\n",
              "      <td>0.260727</td>\n",
              "      <td>0.595065</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Test Accuracy</th>\n",
              "      <td>0.264381</td>\n",
              "      <td>0.590740</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   Our Implementation  Sklearn's Implementation\n",
              "Training Accuracy            0.260727                  0.595065\n",
              "Test Accuracy                0.264381                  0.590740"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TgzZZIYwzZY4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "4d9eccf5-3afb-433d-dff4-99c4c0a4d01e"
      },
      "source": [
        "y_pred = lr.predict(optimal_params)\n",
        "print(sklearn.metrics.r2_score(y_sk, y_pred))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.25654301128603035\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}