02_tensor_basics.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "02_tensor_basics.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4XOZ2VU2ezUI",
        "colab_type": "text"
      },
      "source": [
        "# Pytorch Basics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ny37Dg9pcTKF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " from __future__ import print_function\n",
        " import torch\n",
        "\n",
        " # Everything in pytorch is based on Tensor operations.\n",
        " # A tensor can have different dimensions\n",
        " # so it can be 1d, 2d, or even 3d and higher\n",
        "\n",
        " # scalar, vector, matrix, tensor"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PpLST5AvVW4",
        "colab_type": "code",
        "outputId": "28176620-1757-4162-fc62-853c3f86e220",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('Versions: torch {} and torch.cuda {}'.format(torch.__version__, torch.version.cuda))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Versions: torch 1.4.0 and torch.cuda 10.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7FgGz9MCxsjW",
        "colab_type": "code",
        "outputId": "56d8f43d-69b9-4904-ed20-e331867e8ad1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# To seed the RNG for all devices (both CPU and CUDA)\n",
        "seed = 2277\n",
        "torch.manual_seed(seed)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f1592f50790>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7ofNkvdxxuh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Seed Numpy RNG, incase you are using. \n",
        "np.random.seed(seed)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pT08PZxVcsLp",
        "colab_type": "code",
        "outputId": "75868a5b-a0c2-4ef7-e90f-afd401e34d13",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# torch.empty(size): uninitiallized\n",
        "x = torch.empty(1) # scalar\n",
        "print(x)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([6.5692e-36])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Ahy-64ocsHn",
        "colab_type": "code",
        "outputId": "97cdfe5d-dad0-49ec-d41e-4711fd3fc8d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x = torch.empty(3) # vector, 1D\n",
        "print(x)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([6.5693e-36, 0.0000e+00, 1.4013e-45])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pqN8I9xpdOzb",
        "outputId": "5cedb71b-810e-4079-fda7-fa709efa5992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "x = torch.empty(2,3) # matrix, 2D\n",
        "print(x)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[6.5692e-36, 0.0000e+00, 2.8026e-45],\n",
            "        [0.0000e+00, 1.4013e-45, 0.0000e+00]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8kPipWgHcsDT",
        "colab_type": "code",
        "outputId": "7d3f1209-3b72-4658-bdc9-35161c699255",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "x = torch.empty(2,2,3) # tensor, 3 dimensions\n",
        "#x = torch.empty(2,2,2,3) # tensor, 4 dimensions\n",
        "print(x)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[[6.5696e-36, 0.0000e+00, 4.2039e-45],\n",
            "         [0.0000e+00, 1.4013e-45, 0.0000e+00]],\n",
            "\n",
            "        [[7.3470e+28, 6.1425e+28, 7.1441e+31],\n",
            "         [6.9987e+22, 7.8675e+34, 4.7418e+30]]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ey5Zq2vecr_B",
        "colab_type": "code",
        "outputId": "cc6caf6a-7b47-4db1-ef5b-5cfc4f41a433",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "# torch.rand(size): random numbers [0, 1]\n",
        "x = torch.rand(5, 3)\n",
        "print(x)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.6065, 0.5149, 0.7770],\n",
            "        [0.9356, 0.5546, 0.0135],\n",
            "        [0.2899, 0.0677, 0.9018],\n",
            "        [0.7751, 0.7498, 0.0843],\n",
            "        [0.4354, 0.3761, 0.3090]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MIReLkc-cr8L",
        "colab_type": "code",
        "outputId": "4a047e1d-7cf3-44fe-e182-a79e67349ac1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "# torch.zeros(size), fill with 0\n",
        "# torch.ones(size), fill with 1\n",
        "x = torch.zeros(5, 3)\n",
        "print(x)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QsnYP2VUcr4w",
        "colab_type": "code",
        "outputId": "782e326b-9c96-4890-c86c-429b61094b4f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# check size\n",
        "# size() and shape are same, shape is alias of size(), just to keep it similar to numpy\n",
        "print(x.size())"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([5, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZJ5fA_GU4aRI",
        "colab_type": "code",
        "outputId": "2da19638-b3e9-46e9-e94b-27eae4616500",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Tensor Rank\n",
        "len(x.size())\n",
        "len(x.shape)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "09UailbNcr1i",
        "colab_type": "code",
        "outputId": "e04e1c15-95d2-49bb-8fa7-a1975a9b158d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# check data type\n",
        "print(x.dtype)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.float32\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3JeNvcDrdelc",
        "colab_type": "code",
        "outputId": "281b99aa-dea8-4873-8ca1-5e867c9d7e85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "# specify types, float32 default\n",
        "x = torch.zeros(5, 3, dtype=torch.float16)\n",
        "print(x)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.],\n",
            "        [0., 0., 0.]], dtype=torch.float16)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-GBiFsKrdehQ",
        "colab_type": "code",
        "outputId": "a990f97a-a5b6-48fc-a747-64f12bb19b79",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# check type\n",
        "print(x.dtype)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.float16\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CWMA34uZdeeg",
        "colab_type": "code",
        "outputId": "a608dce0-0d6d-4ee4-9d0a-10e11bc56ac8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# construct from data\n",
        "x = torch.tensor([5.5, 3])\n",
        "print(x.size())"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v11pDXbC4F9D",
        "colab_type": "code",
        "outputId": "69e5d153-7247-4265-9892-038e8d1f37ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Rank\n",
        "len(x.shape)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14ATgX1edecB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# requires_grad argument\n",
        "# This will tell pytorch that it will need to calculate the gradients for this tensor\n",
        "# later in your optimization steps\n",
        "# i.e. this is a variable in your model that you want to optimize\n",
        "x = torch.tensor([5.5, 3], requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ieOgodO3deZH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Operations\n",
        "y = torch.rand(2, 2)\n",
        "x = torch.rand(2, 2)\n",
        "\n",
        "# elementwise addition\n",
        "z = x + y\n",
        "#z = torch.add(x,y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ufpy-1_QdeUl",
        "colab_type": "code",
        "outputId": "8e89b8a9-51ee-436d-e0b7-1ddfce29aa54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# in place addition, everythin with a trailing underscore is an inplace operation\n",
        "# i.e. it will modify the variable\n",
        "y.add_(x)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.8439, 0.9812],\n",
              "        [1.2357, 0.4613]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T90PKyfDdeRM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# substraction\n",
        "z = x - y\n",
        "z = torch.sub(x, y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_76F0_53deNg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# multiplication\n",
        "z = x * y\n",
        "z = torch.mul(x,y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eTBtOymjeK4F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# division\n",
        "z = x / y\n",
        "z = torch.div(x,y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17mfvlTJeK2I",
        "colab_type": "code",
        "outputId": "01185994-fcb6-41a5-c90b-4f8f3dda6bc2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "# Slicing\n",
        "x = torch.rand(5,3)\n",
        "print(x)\n",
        "print(x[:, 0]) # all rows, column 0\n",
        "print(x[1, :]) # row 1, all columns\n",
        "print(x[1,1]) # element at 1, 1"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.9238, 0.9162, 0.9575],\n",
            "        [0.5560, 0.1932, 0.1462],\n",
            "        [0.3941, 0.4508, 0.5634],\n",
            "        [0.0057, 0.0411, 0.5529],\n",
            "        [0.0978, 0.3070, 0.9777]])\n",
            "tensor([0.9238, 0.5560, 0.3941, 0.0057, 0.0978])\n",
            "tensor([0.5560, 0.1932, 0.1462])\n",
            "tensor(0.1932)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bmWQVeP3eKyU",
        "colab_type": "code",
        "outputId": "e7623453-d660-438f-818b-9e3eef336fab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Get the actual value if only 1 element in your tensor\n",
        "print(x[1,1].item())\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.19320964813232422\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Wqo_wxzeKvW",
        "colab_type": "code",
        "outputId": "6177e1d4-3bff-4b88-9211-eeb2b6211373",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Reshape with torch.view()\n",
        "x = torch.randn(4, 4)\n",
        "y = x.view(16)\n",
        "z = x.view(-1, 8)  # the size -1 is inferred from other dimensions\n",
        "# if -1 it pytorch will automatically determine the necessary size\n",
        "print(x.size(), y.size(), z.size())"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4, 4]) torch.Size([16]) torch.Size([2, 8])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jZ--bFp2eKs8",
        "colab_type": "code",
        "outputId": "e38cdef4-d031-4605-d9c5-7ba8960ec5dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Numpy\n",
        "# Converting a Torch Tensor to a NumPy array and vice versa is very easy\n",
        "a = torch.ones(5)\n",
        "print(a)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1.])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IZ7_q099eKp_",
        "colab_type": "code",
        "outputId": "62de5eae-f79f-4921-984d-2d6a390aaab2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# torch to numpy with .numpy()\n",
        "b = a.numpy()\n",
        "print(b)\n",
        "print(type(b))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1. 1. 1. 1. 1.]\n",
            "<class 'numpy.ndarray'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3Qdwgf4eKmK",
        "colab_type": "code",
        "outputId": "9d6cb016-0946-4582-bb62-4a02f1a1858e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Carful: If the Tensor is on the CPU (not the GPU),\n",
        "# both objects will share the same memory location, so changing one\n",
        "# will also change the other\n",
        "a.add_(1)\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([2., 2., 2., 2., 2.])\n",
            "[2. 2. 2. 2. 2.]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAkPKrXBej-f",
        "colab_type": "code",
        "outputId": "5bf7f3b4-d64a-4c65-819f-37999816eb7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# numpy to torch with .from_numpy(x)\n",
        "import numpy as np\n",
        "a = np.ones(5)\n",
        "b = torch.from_numpy(a)\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1. 1. 1. 1. 1.]\n",
            "tensor([1., 1., 1., 1., 1.], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ut6dmgysej5o",
        "colab_type": "code",
        "outputId": "3ef7f432-5122-4534-e61c-2c11f64a15be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# again be careful when modifying\n",
        "a += 1\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2. 2. 2. 2. 2.]\n",
            "tensor([2., 2., 2., 2., 2.], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kj04SSVPej20",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# by default all tensors are created on the CPU,\n",
        "# but you can also move them to the GPU (only if it's available )\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")          # a CUDA device object\n",
        "    y = torch.ones_like(x, device=device)  # directly create a tensor on GPU\n",
        "    x = x.to(device)                       # or just use strings ``.to(\"cuda\")``\n",
        "    z = x + y\n",
        "    # z = z.numpy() # not possible because numpy cannot handle GPU tenors\n",
        "    # move to CPU again\n",
        "    z.to(\"cpu\")       # ``.to`` can also change dtype together!\n",
        "    # z = z.numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ghHl6yKL7wo",
        "colab_type": "text"
      },
      "source": [
        "## CUDA Know How"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4RSU943MBXh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "691f2f98-3872-476a-d224-1ef298dcbcee"
      },
      "source": [
        "torch.cuda.is_available()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4BkkbX8JMQzn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "90b0a70f-90a8-4a13-c1bf-279ae98f62fb"
      },
      "source": [
        "# Get to know CUDA device(s)\n",
        "\n",
        "print(torch.cuda.device_count())\n",
        "print(torch.cuda.get_device_name())\n",
        "print(torch.cuda.get_device_capability())\n",
        "print(torch.cuda.get_device_properties(0))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "Tesla P4\n",
            "(6, 1)\n",
            "_CudaDeviceProperties(name='Tesla P4', major=6, minor=1, total_memory=7611MB, multi_processor_count=20)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8Bk6dAfMF5Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Seeding CUDA\n",
        "\n",
        "torch.cuda.manual_seed(seed)\n",
        "\n",
        "torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "torch.backends.cudnn.deterministic=True"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}