prepare_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "prepare_data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "lG5yxTxARRJ8"
      },
      "source": [
        "import urllib.request\n",
        "import numpy as np\n",
        "import subprocess\n",
        "import os\n",
        "from google.colab import files\n",
        "import shutil\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GMeq7D7ERzwI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "outputId": "56930527-5205-4886-b5f7-457666b99b84"
      },
      "source": [
        "from google.colab import drive\n",
        "root_dir = '/content/drive/'\n",
        "drive.mount(root_dir, force_remount = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hDGIsOIhRfOK"
      },
      "source": [
        "Download the Data:\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzXJ_XvBRhIG"
      },
      "source": [
        "def download(classes):\n",
        "    link = 'https://storage.googleapis.com/quickdraw_dataset/full/numpy_bitmap'\n",
        "    base_dir = root_dir + 'My Drive/CS7643_Final_Project/Dataset'\n",
        "    print('Downloading ...')\n",
        "    for c in classes:\n",
        "        cname = c.replace(\" \", \"%20\") # (Google cloud links replace spaces with 20%)\n",
        "        print(f'{link}/{cname}.npy')\n",
        "        urllib.request.urlretrieve(f'{link}/{cname}.npy', f'{base_dir}/{c}.npy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3jsoh4LGGtW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "2975ff4d-bd40-4a86-916b-42006576e5fc"
      },
      "source": [
        "download(['flamingo', 'sheep'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading ...\n",
            "https://storage.googleapis.com/quickdraw_dataset/full/numpy_bitmap/flamingo.npy\n",
            "https://storage.googleapis.com/quickdraw_dataset/full/numpy_bitmap/sheep.npy\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aaP-Rq17RkGt"
      },
      "source": [
        "Split Data into training, test and validation, and save it:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8ATINdPRn6w"
      },
      "source": [
        "def prepare_data():\n",
        "    \"\"\"\n",
        "    Split data into training, validation, and testing data\n",
        "    \"\"\"\n",
        "    base_dir = root_dir + 'My Drive/CS7643_Final_Project/Dataset'\n",
        "    file_names = os.listdir(base_dir)\n",
        "    \n",
        "    x_train, y_train = np.empty([0, 784]), np.empty([0])\n",
        "    x_valid, y_valid = np.empty([0, 784]), np.empty([0])\n",
        "    x_test, y_test = np.empty([0, 784]), np.empty([0])\n",
        "    \n",
        "    for idx, fname in enumerate(file_names):\n",
        "        print(fname)\n",
        "        \n",
        "        data = np.load(f'{base_dir}/{fname}', allow_pickle=True)     \n",
        "        labels = np.full(data.shape[0], idx)\n",
        "        \n",
        "        x_train = np.concatenate((x_train, data[:8400, :]), axis=0)\n",
        "        y_train = np.append(y_train, labels[:8400])\n",
        "\n",
        "        x_valid = np.concatenate((x_valid, data[8400: 12000, :]), axis=0)\n",
        "        y_valid = np.append(y_valid, labels[8400: 12000])\n",
        "\n",
        "        x_test = np.concatenate((x_test, data[12000: 15000, :]), axis=0)\n",
        "        y_test = np.append(y_test, labels[12000: 15000])\n",
        "\n",
        "\n",
        "    N = y_train.shape[0]\n",
        "    # randomize training dataset\n",
        "    rand = np.random.permutation(N)\n",
        "    x_train, y_train = x_train[rand, :], y_train[rand]\n",
        "\n",
        "    print(f'{len(y_train)} training files, {len(y_valid)} validation files, {len(y_test)} testing files')\n",
        "\n",
        "    os.mkdir(root_dir + 'My Drive/CS7643_Final_Project/Train')\n",
        "    np.savez_compressed(root_dir + 'My Drive/CS7643_Final_Project/Train/Train', data=x_train, target=y_train)\n",
        "    os.mkdir(root_dir + 'My Drive/CS7643_Final_Project/Validation')\n",
        "    np.savez_compressed(root_dir + 'My Drive/CS7643_Final_Project/Validation/Validation', data=x_valid, target=y_valid)\n",
        "    os.mkdir(root_dir + 'My Drive/CS7643_Final_Project/Test')\n",
        "    np.savez_compressed(root_dir + 'My Drive/CS7643_Final_Project/Test/Test', data=x_test, target=y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sn31eFKMR5s8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "6bc2e5b3-661e-4908-cbf4-c3cda684c7fd"
      },
      "source": [
        "prepare_data()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tree.npy\n",
            "t-shirt.npy\n",
            "ice cream.npy\n",
            "fish.npy\n",
            "face.npy\n",
            "car.npy\n",
            "bowtie.npy\n",
            "apple.npy\n",
            "flamingo.npy\n",
            "sheep.npy\n",
            "84000 training files, 36000 validation files, 30000 testing files\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RknXBtKqach4"
      },
      "source": [
        "def load_data(batch_size = 128):\n",
        "    \"\"\"\n",
        "    Returns a list of PyTorch DataLoaders in order of train, validation, and test loaders\n",
        "    \"\"\"\n",
        "    dirnames = [root_dir + 'My Drive/CS7643_Final_Project/Train/', root_dir + 'My Drive/CS7643_Final_Project/Validation/', root_dir + 'My Drive/CS7643_Final_Project/Test/']\n",
        "    loaders = []\n",
        "    for dirname in dirnames:\n",
        "        dataname = os.listdir(dirname)[0]\n",
        "        xs = np.load(dirname + dataname)['data'].reshape(-1, 28, 28)\n",
        "        ys = np.load(dirname + dataname)['target']\n",
        "        print(f'Loading {dataname} into Pytorch Dataloader ... Xs shape: {xs.shape}, Ys shape: {ys.shape}')\n",
        "        loaders.append(DataLoader(TensorDataset(Tensor(xs), Tensor(ys)), batch_size = batch_size, shuffle=True))\n",
        "    \n",
        "    return loaders"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}