01_Pytorch_Fundamentals.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6453b774-2630-41d8-9696-7cb57127fdb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "import torchinfo, torchmetrics, torch\n",
    "\n",
    "# Check for GPU (should return True)\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4846c955-962d-4f31-bfcd-6f830841cb41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU 0: NVIDIA GeForce RTX 4070 Laptop GPU (UUID: GPU-98579472-7258-af99-7b4b-bbcd2dce65c8)\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi -L"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f2dd7ec-97e0-4a75-8054-14379e2c8aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import sklearn\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c80c8f18-20ee-4c90-aa88-3ebd38d393a3",
   "metadata": {},
   "source": [
    "# Introduction to Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc9377a0-7fa1-4e87-855d-b694328b23cc",
   "metadata": {},
   "source": [
    "Creating tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48096f43-a388-42c3-9b12-fe3d1805fcd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scalar = torch.tensor(6)\n",
    "scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edcaa766-d9a6-41ea-9d42-72b4b76bf721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scalar.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "63696b1e-0cab-463e-93b8-54aa3ebf688a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scalar.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a7dffc12-66d9-48d5-9c79-c56c88cabc5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([6, 6])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Vector\n",
    "vector = torch.tensor([6,6])\n",
    "vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a8a6d3b-06cc-47f3-88d3-ef4d606c6820",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([2]), 1)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector.shape, vector.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "06ebefe3-e11a-4201-b04a-7bb692a60a4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1, 6],\n",
       "         [9, 6]]),\n",
       " 2,\n",
       " torch.Size([2, 2]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Matrix \n",
    "MATRIX = torch.tensor([[1, 6],\n",
    "                      [9, 6]])\n",
    "\n",
    "MATRIX, MATRIX.ndim, MATRIX.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "439b31a9-8696-4b5a-a539-e2e9d01dcd38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1, 2, 3],\n",
       "         [4, 5, 6],\n",
       "         [7, 8, 9]]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TENSOR\n",
    "TENSOR = torch.tensor([[[1,2,3],\n",
    "                      [4,5,6],\n",
    "                      [7,8,9]]])\n",
    "\n",
    "TENSOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0ee3bf6b-64ab-4756-a16b-5ec3275b2af1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, torch.Size([1, 3, 3]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TENSOR.ndim, TENSOR.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8b03361-7f52-4d5d-881d-affd11850d2a",
   "metadata": {},
   "source": [
    "# Random Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "422808f5-57b0-4295-aed8-c08fb806abf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9961, 0.6366, 0.4546, 0.9491],\n",
       "        [0.4373, 0.5096, 0.8026, 0.8199],\n",
       "        [0.9896, 0.9867, 0.9539, 0.1004]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a random tensor of size (3, 4)\n",
    "random_tensor = torch.rand(3,4)\n",
    "random_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "65f8006f-35f8-4416-a3b3-832045e7ba45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3, 4]), 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_tensor.shape, random_tensor.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "920e104f-ad3c-48cf-aee1-55bd6acafdfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([224, 224, 3]), 3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a random tensor with similar shape as image\n",
    "random_image_size_tensor = torch.rand(224, 224, 3)\n",
    "random_image_size_tensor.shape, random_image_size_tensor.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da447d72-6a80-4ee4-90b3-4eb5f3af7fe7",
   "metadata": {},
   "source": [
    "# Zeros and ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9bfba92c-d65f-46bc-a6e4-d4e47455ad05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0.]]),\n",
       " torch.float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor of all zeros\n",
    "zeros = torch.zeros(size=(3,4))\n",
    "zeros, zeros.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e343a2c5-a3e5-4faf-b231-93ca9d3d933f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 1., 1., 1.],\n",
       "         [1., 1., 1., 1.],\n",
       "         [1., 1., 1., 1.]]),\n",
       " torch.float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor of all ones\n",
    "ones = torch.ones(size=(3,4))\n",
    "ones, zeros.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71615213-45da-4c83-b6f8-c2692ea9a06a",
   "metadata": {},
   "source": [
    "# creating a range and tensors like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2ed065bb-d150-49f9-946a-b7aefa31ee36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10]), torch.int64)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use torch.arange()\n",
    "# zero_to_ten_deprecated = torch.range(0, 10)\n",
    "\n",
    "# create a range of values 0 to 10\n",
    "zero_to_ten = torch.arange(0, 11, 1)\n",
    "zero_to_ten, zero_to_ten.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "743ad072-a25b-4687-97cc-c07789feac1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Can also create a tensor of zeros similar to another tensor\n",
    "ten_zeros = torch.zeros_like(zero_to_ten)\n",
    "ten_zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0248a947-a91e-4e43-8532-f68f263e9116",
   "metadata": {},
   "source": [
    "# Tensor Datatypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ea853b69-d816-4f1a-aed0-a9cd15da29d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3]), torch.float32, device(type='cpu'))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Default datatype for tensors is float32\n",
    "float_32_tensor = torch.tensor([3.4, 6.0, 7.2],\n",
    "                               dtype=None, # defaults to None, which is torch.float32 or whatever datatype is passed\n",
    "                               device=None, # defaults to None, which uses the default tensor type\n",
    "                               requires_grad=False) # if True, operations performed on the tensor are recorded \n",
    "\n",
    "float_32_tensor.shape, float_32_tensor.dtype, float_32_tensor.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0022b069-4bfe-4955-a1f2-59d903acebc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([3., 6., 9.], dtype=torch.float16),\n",
       " tensor([3.1992, 6.0000, 8.7969], dtype=torch.float16))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_16_tensor = torch.tensor([3.0, 6.0, 9.0],\n",
    "                               dtype=torch.float16)\n",
    "\n",
    "float_16_tensor_half = torch.tensor([3.2, 6.0, 8.8],\n",
    "                                   dtype=torch.half)  # torch.float16 and torch.half are same\n",
    "\n",
    "float_16_tensor, float_16_tensor_half"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bd5e22a-8b65-4b0f-a2a6-81964d96feb2",
   "metadata": {},
   "source": [
    "## Getting information from tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "179f32aa-9589-463d-aaa3-105da85095a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.7747, 0.0756, 0.6883, 0.0034],\n",
      "        [0.8236, 0.7687, 0.8651, 0.2750],\n",
      "        [0.0686, 0.3589, 0.8538, 0.8891],\n",
      "        [0.1563, 0.7846, 0.6559, 0.1068]])\n",
      "Shape of tensor: torch.Size([4, 4])\n",
      "Datatype of tensor: torch.float32\n",
      "Device tensor is stored on: cpu\n"
     ]
    }
   ],
   "source": [
    "# Create a tensor\n",
    "some_tensor = torch.rand(4, 4)\n",
    "\n",
    "# Find out details about it\n",
    "print(some_tensor)\n",
    "print(f\"Shape of tensor: {some_tensor.shape}\")\n",
    "print(f\"Datatype of tensor: {some_tensor.dtype}\")\n",
    "print(f\"Device tensor is stored on: {some_tensor.device}\") # will default to CPU"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1ff4806-4ccb-4319-b7d0-edde8530b8a2",
   "metadata": {},
   "source": [
    "## Manipulating Tensors (tensor operations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "46ea4060-b578-44f8-818a-5fc552b0ba5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([11, 12, 13])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a tensor\n",
    "tensor = torch.tensor([1,2,3])\n",
    "tensor+10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d2f21b57-06a2-4f19-81ff-75e2dc320dfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 20, 30])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Multiply tensor by 10\n",
    "tensor = tensor * 10\n",
    "tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f993322f-613d-44a9-b3b1-f3530044cd67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1,  9, 19])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor - 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cbcf20d9-7ecf-46a6-9c05-22d5adae40c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([100, 200, 300])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mul(tensor, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c9c42f21-630b-4a24-8b49-75bed34afe4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([140, 200, 260])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(tensor, torch.tensor([[1,2,3],\n",
    "                                  [2,3,4],\n",
    "                                  [3,4,5]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3025a87c-b7a3-4aeb-aecf-af69b3a41560",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 3])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor([[1,2,3],[2,3,4],[3,4,5]]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3f125a8b-615c-45a2-ac90-6c8a2b11fb4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shapes need to be in the right way  \n",
    "tensor_A = torch.tensor([[1, 2],\n",
    "                         [3, 4],\n",
    "                         [5, 6]], dtype=torch.float32)\n",
    "\n",
    "tensor_B = torch.tensor([[7, 10],\n",
    "                         [8, 11], \n",
    "                         [9, 12]], dtype=torch.float32)\n",
    "\n",
    "# torch.matmul(tensor_A, tensor_B) # (this will error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1066ab05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original shapes: tensor_A = torch.Size([3, 2]), tensor_B = torch.Size([3, 2])\n",
      "\n",
      "New shapes: tensor_A = torch.Size([3, 2]) (same as above), tensor_B.T = torch.Size([2, 3])\n",
      "\n",
      "Multiplying: torch.Size([3, 2]) * torch.Size([2, 3]) <- inner dimensions match\n",
      "\n",
      "Output:\n",
      "\n",
      "tensor([[ 27.,  30.,  33.],\n",
      "        [ 61.,  68.,  75.],\n",
      "        [ 95., 106., 117.]])\n",
      "\n",
      "Output shape: torch.Size([3, 3])\n"
     ]
    }
   ],
   "source": [
    "# The operation works when tensor_B is transposed\n",
    "print(f\"Original shapes: tensor_A = {tensor_A.shape}, tensor_B = {tensor_B.shape}\\n\")\n",
    "print(f\"New shapes: tensor_A = {tensor_A.shape} (same as above), tensor_B.T = {tensor_B.T.shape}\\n\")\n",
    "print(f\"Multiplying: {tensor_A.shape} * {tensor_B.T.shape} <- inner dimensions match\\n\")\n",
    "print(\"Output:\\n\")\n",
    "output = torch.matmul(tensor_A, tensor_B.T)\n",
    "print(output) \n",
    "print(f\"\\nOutput shape: {output.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "67ba0836",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input shape: torch.Size([2, 3])\n",
      "\n",
      "Output:\n",
      "tensor([[0.9332, 0.8805, 3.0149, 1.5545, 1.8186, 2.0634],\n",
      "        [1.7186, 1.4009, 3.5818, 1.7408, 2.6017, 2.5123]],\n",
      "       grad_fn=<AddmmBackward0>)\n",
      "\n",
      "Output shape: torch.Size([2, 6])\n"
     ]
    }
   ],
   "source": [
    "# Since the linear layer starts with a random weights matrix, let's make it reproducible (more on this later)\n",
    "torch.manual_seed(42)\n",
    "# This uses matrix multiplication\n",
    "linear = torch.nn.Linear(in_features=3, # in_features = matches inner dimension of input \n",
    "                         out_features=6) # out_features = describes outer value \n",
    "x = tensor_A.T\n",
    "output = linear(x)\n",
    "print(f\"Input shape: {x.shape}\\n\")\n",
    "print(f\"Output:\\n{output}\\n\\nOutput shape: {output.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18fb5457",
   "metadata": {},
   "source": [
    "### Finding the min, max, mean, sum, etc (aggregation)\n",
    "\n",
    "Now we've seen a few ways to manipulate tensors, let's run through a few ways to aggregate them (go from more values to less values).\n",
    "\n",
    "First we'll create a tensor and then find the max, min, mean and sum of it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "62970e08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0, 10, 20, 30, 40, 50, 60, 70, 80, 90])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a tensor\n",
    "x = torch.arange(0,100,10)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "246f1587",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum: 0\n",
      "Maximum: 90\n",
      "Mean: 45.0\n",
      "Sum: 450\n"
     ]
    }
   ],
   "source": [
    "print(f\"Minimum: {x.min()}\")\n",
    "print(f\"Maximum: {x.max()}\")\n",
    "# print(f\"Mean: {x.mean()}\") # this will error\n",
    "print(f\"Mean: {x.type(torch.float32).mean()}\") # won't work without float datatype\n",
    "print(f\"Sum: {x.sum()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4c5e10a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(90), tensor(0), tensor(45.), tensor(450))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.max(x), torch.min(x), torch.mean(x.type(torch.float32)), torch.sum(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fde22244",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor: tensor([10, 20, 30, 40, 50, 60, 70, 80, 90])\n",
      "Index where max value occurs: 8\n",
      "Index where min value occurs: 0\n"
     ]
    }
   ],
   "source": [
    "# Create a tensor\n",
    "tensor = torch.arange(10, 100, 10)\n",
    "print(f\"Tensor: {tensor}\")\n",
    "\n",
    "# Returns index of max and min values\n",
    "print(f\"Index where max value occurs: {tensor.argmax()}\")\n",
    "print(f\"Index where min value occurs: {tensor.argmin()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "badfdacf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(3), tensor(1))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.argmin(torch.tensor([1,5,3,0])), torch.argmax(torch.tensor([1,5,3,0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f14dd09e",
   "metadata": {},
   "source": [
    "### Change tensor datatype\n",
    "\n",
    "As mentioned, a common issue with deep learning operations is having your tensors in different datatypes.\n",
    "\n",
    "If one tensor is in `torch.float64` and another is in `torch.float32`, you might run into some errors.\n",
    "\n",
    "But there's a fix.\n",
    "\n",
    "You can change the datatypes of tensors using [`torch.Tensor.type(dtype=None)`](https://pytorch.org/docs/stable/generated/torch.Tensor.type.html) where the `dtype` parameter is the datatype you'd like to use.\n",
    "\n",
    "First we'll create a tensor and check its datatype (the default is `torch.float32`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f48f115d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a tensor and check its datatype\n",
    "tensor_default = torch.arange(10., 100., 10.)\n",
    "tensor_default.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "dd73798b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10., 20., 30., 40., 50., 60., 70., 80., 90.], dtype=torch.float16)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a float16 tensor\n",
    "tensor_float16 = tensor_default.type(torch.float16)\n",
    "tensor_float16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "61f3f779",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 20, 30, 40, 50, 60, 70, 80, 90], dtype=torch.int8)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create an int8 tensor\n",
    "tensor_int8 = tensor.type(torch.int8)\n",
    "tensor_int8"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f909119",
   "metadata": {},
   "source": [
    "### Reshaping, stacking, squeezing and unsqueezing\n",
    "\n",
    "Often times you'll want to reshape or change the dimensions of your tensors without actually changing the values inside them.\n",
    "\n",
    "To do so, some popular methods are:\n",
    "\n",
    "| Method | One-line description |\n",
    "| ----- | ----- |\n",
    "| [`torch.reshape(input, shape)`](https://pytorch.org/docs/stable/generated/torch.reshape.html#torch.reshape) | Reshapes `input` to `shape` (if compatible), can also use `torch.Tensor.reshape()`. |\n",
    "| [`Tensor.view(shape)`](https://pytorch.org/docs/stable/generated/torch.Tensor.view.html) | Returns a view of the original tensor in a different `shape` but shares the same data as the original tensor. |\n",
    "| [`torch.stack(tensors, dim=0)`](https://pytorch.org/docs/1.9.1/generated/torch.stack.html) | Concatenates a sequence of `tensors` along a new dimension (`dim`), all `tensors` must be same size. |\n",
    "| [`torch.squeeze(input)`](https://pytorch.org/docs/stable/generated/torch.squeeze.html) | Squeezes `input` to remove all the dimenions with value `1`. |\n",
    "| [`torch.unsqueeze(input, dim)`](https://pytorch.org/docs/1.9.1/generated/torch.unsqueeze.html) | Returns `input` with a dimension value of `1` added at `dim`. | \n",
    "| [`torch.permute(input, dims)`](https://pytorch.org/docs/stable/generated/torch.permute.html) | Returns a *view* of the original `input` with its dimensions permuted (rearranged) to `dims`. | \n",
    "\n",
    "Why do any of these?\n",
    "\n",
    "Because deep learning models (neural networks) are all about manipulating tensors in some way. And because of the rules of matrix multiplication, if you've got shape mismatches, you'll run into errors. These methods help you make sure the right elements of your tensors are mixing with the right elements of other tensors. \n",
    "\n",
    "Let's try them out.\n",
    "\n",
    "First, we'll create a tensor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "852dffda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 2., 3., 4., 5., 6., 7.]), torch.Size([7]))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a tensor\n",
    "import torch\n",
    "x = torch.arange(1., 8.)\n",
    "x, x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "377ca6b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 2., 3., 4., 5., 6., 7.]]), torch.Size([1, 7]))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add an extra dimension\n",
    "x_reshaped = x.reshape(1, 7)\n",
    "x_reshaped, x_reshaped.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "e1a15119",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 2., 3., 4., 5., 6., 7.]]), torch.Size([1, 7]))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = x.view(1, 7)\n",
    "z, z.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "04585622",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[5., 2., 3., 4., 5., 6., 7.]]), tensor([5., 2., 3., 4., 5., 6., 7.]))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Changing z changes x\n",
    "z[:, 0] = 5\n",
    "z, x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2b8f7e81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[5., 2., 3., 4., 5., 6., 7.],\n",
       "        [5., 2., 3., 4., 5., 6., 7.],\n",
       "        [5., 2., 3., 4., 5., 6., 7.],\n",
       "        [5., 2., 3., 4., 5., 6., 7.]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Stack tensors on top of each other\n",
    "x_stacked = torch.stack([x, x, x, x], dim=0) # try changing dim to dim=1 and see what happens\n",
    "x_stacked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "692386f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previous tensor: tensor([[5., 2., 3., 4., 5., 6., 7.]])\n",
      "Previous shape: torch.Size([1, 7])\n",
      "\n",
      "New tensor: tensor([5., 2., 3., 4., 5., 6., 7.])\n",
      "New shape: torch.Size([7])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Previous tensor: {x_reshaped}\")\n",
    "print(f\"Previous shape: {x_reshaped.shape}\")\n",
    "\n",
    "# Remove extra dimension from x_reshaped\n",
    "x_squeezed = x_reshaped.squeeze()\n",
    "print(f\"\\nNew tensor: {x_squeezed}\")\n",
    "print(f\"New shape: {x_squeezed.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "09d7d3af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previous tensor: tensor([5., 2., 3., 4., 5., 6., 7.])\n",
      "Previous shape: torch.Size([7])\n",
      "\n",
      "New tensor: tensor([[5., 2., 3., 4., 5., 6., 7.]])\n",
      "New shape: torch.Size([1, 7])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Previous tensor: {x_squeezed}\")\n",
    "print(f\"Previous shape: {x_squeezed.shape}\")\n",
    "\n",
    "## Add an extra dimension with unsqueeze\n",
    "x_unsqueezed = x_squeezed.unsqueeze(dim=0)\n",
    "print(f\"\\nNew tensor: {x_unsqueezed}\")\n",
    "print(f\"New shape: {x_unsqueezed.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "2cd8189a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previous shape: torch.Size([224, 224, 3])\n",
      "New shape: torch.Size([3, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "# Create tensor with specific shape\n",
    "x_original = torch.rand(size=(224, 224, 3))\n",
    "\n",
    "# Permute the original tensor to rearrange the axis order\n",
    "x_permuted = x_original.permute(2, 0, 1) # shifts axis 0->1, 1->2, 2->0\n",
    "\n",
    "print(f\"Previous shape: {x_original.shape}\")\n",
    "print(f\"New shape: {x_permuted.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4916f6b",
   "metadata": {},
   "source": [
    "## Indexing (selecting data from tensors)\n",
    "\n",
    "Sometimes you'll want to select specific data from tensors (for example, only the first column or second row).\n",
    "\n",
    "To do so, you can use indexing.\n",
    "\n",
    "If you've ever done indexing on Python lists or NumPy arrays, indexing in PyTorch with tensors is very similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c8ff20eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[1, 2, 3],\n",
       "          [4, 5, 6],\n",
       "          [7, 8, 9]]]),\n",
       " torch.Size([1, 3, 3]))"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Create a tensor \n",
    "import torch\n",
    "x = torch.arange(1, 10).reshape(1, 3, 3)\n",
    "x, x.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ab398d2",
   "metadata": {},
   "source": [
    "\n",
    "Indexing values goes outer dimension -> inner dimension (check out the square brackets).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "abcf2b57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First square bracket:\n",
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6],\n",
      "        [7, 8, 9]])\n",
      "Second square bracket: tensor([1, 2, 3])\n",
      "Third square bracket: 1\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Let's index bracket by bracket\n",
    "print(f\"First square bracket:\\n{x[0]}\") \n",
    "print(f\"Second square bracket: {x[0][0]}\") \n",
    "print(f\"Third square bracket: {x[0][0][0]}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69286de9",
   "metadata": {},
   "source": [
    "\n",
    "You can also use `:` to specify \"all values in this dimension\" and then use a comma (`,`) to add another dimension.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "fbc22182",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Get all values of 0th dimension and the 0 index of 1st dimension\n",
    "x[:, 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "0b5c86f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 5, 8]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Get all values of 0th & 1st dimensions but only index 1 of 2nd dimension\n",
    "x[:, :, 1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "fbe25614",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Get all values of the 0 dimension but only the 1 index value of the 1st and 2nd dimension\n",
    "x[:, 1, 1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "895cc95c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Get index 0 of 0th and 1st dimension and all values of 2nd dimension \n",
    "x[0, 0, :] # same as x[0][0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a7c38b7",
   "metadata": {},
   "source": [
    "## PyTorch tensors & NumPy\n",
    "\n",
    "Since NumPy is a popular Python numerical computing library, PyTorch has functionality to interact with it nicely.  \n",
    "\n",
    "The two main methods you'll want to use for NumPy to PyTorch (and back again) are: \n",
    "* [`torch.from_numpy(ndarray)`](https://pytorch.org/docs/stable/generated/torch.from_numpy.html) - NumPy array -> PyTorch tensor. \n",
    "* [`torch.Tensor.numpy()`](https://pytorch.org/docs/stable/generated/torch.Tensor.numpy.html) - PyTorch tensor -> NumPy array.\n",
    "\n",
    "Let's try them out."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e0629d80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1., 2., 3., 4., 5., 6., 7.]),\n",
       " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# NumPy array to tensor\n",
    "import torch\n",
    "import numpy as np\n",
    "array = np.arange(1.0, 8.0)\n",
    "tensor = torch.from_numpy(array)\n",
    "array, tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80bc130c",
   "metadata": {},
   "source": [
    "\n",
    "> **Note:** By default, NumPy arrays are created with the datatype `float64` and if you convert it to a PyTorch tensor, it'll keep the same datatype (as above). \n",
    ">\n",
    "> However, many PyTorch calculations default to using `float32`. \n",
    "> \n",
    "> So if you want to convert your NumPy array (float64) -> PyTorch tensor (float64) -> PyTorch tensor (float32), you can use `tensor = torch.from_numpy(array).type(torch.float32)`.\n",
    "\n",
    "Because we reassigned `tensor` above, if you change the tensor, the array stays the same.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "e0246787",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2., 3., 4., 5., 6., 7., 8.]),\n",
       " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Change the array, keep the tensor\n",
    "array = array + 1\n",
    "array, tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bec6064",
   "metadata": {},
   "source": [
    "\n",
    "And if you want to go from PyTorch tensor to NumPy array, you can call `tensor.numpy()`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6f705f0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 1., 1., 1., 1., 1., 1.]),\n",
       " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor to NumPy array\n",
    "tensor = torch.ones(7) # create a tensor of ones with dtype=float32\n",
    "numpy_tensor = tensor.numpy() # will be dtype=float32 unless changed\n",
    "tensor, numpy_tensor\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1ab2ee9",
   "metadata": {},
   "source": [
    "\n",
    "And the same rule applies as above, if you change the original `tensor`, the new `numpy_tensor` stays the same.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "0b30cda4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([2., 2., 2., 2., 2., 2., 2.]),\n",
       " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Change the tensor, keep the array the same\n",
    "tensor = tensor + 1\n",
    "tensor, numpy_tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eccbce57",
   "metadata": {},
   "source": [
    "## Reproducibility (trying to take the random out of random)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "a0ad91ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor A:\n",
      "tensor([[0.5745, 0.9200, 0.3230, 0.8613],\n",
      "        [0.0919, 0.3102, 0.9536, 0.6002],\n",
      "        [0.0351, 0.6826, 0.3743, 0.5220]])\n",
      "\n",
      "Tensor B:\n",
      "tensor([[0.1336, 0.9666, 0.9754, 0.8474],\n",
      "        [0.8988, 0.1105, 0.4563, 0.9719],\n",
      "        [0.3968, 0.1496, 0.4743, 0.9973]])\n",
      "\n",
      "Does Tensor A equal Tensor B? (anywhere)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[False, False, False, False],\n",
       "        [False, False, False, False],\n",
       "        [False, False, False, False]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# Create two random tensors\n",
    "random_tensor_A = torch.rand(3, 4)\n",
    "random_tensor_B = torch.rand(3, 4)\n",
    "\n",
    "print(f\"Tensor A:\\n{random_tensor_A}\\n\")\n",
    "print(f\"Tensor B:\\n{random_tensor_B}\\n\")\n",
    "print(f\"Does Tensor A equal Tensor B? (anywhere)\")\n",
    "random_tensor_A == random_tensor_B\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "48d9227c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor C:\n",
      "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
      "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
      "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
      "\n",
      "Tensor D:\n",
      "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
      "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
      "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
      "\n",
      "Does Tensor C equal Tensor D? (anywhere)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[True, True, True, True],\n",
       "        [True, True, True, True],\n",
       "        [True, True, True, True]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import random\n",
    "\n",
    "# # Set the random seed\n",
    "RANDOM_SEED=42 # try changing this to different values and see what happens to the numbers below\n",
    "torch.manual_seed(seed=RANDOM_SEED) \n",
    "random_tensor_C = torch.rand(3, 4)\n",
    "\n",
    "# Have to reset the seed every time a new rand() is called \n",
    "# Without this, tensor_D would be different to tensor_C \n",
    "torch.random.manual_seed(seed=RANDOM_SEED) # try commenting this line out and seeing what happens\n",
    "random_tensor_D = torch.rand(3, 4)\n",
    "\n",
    "print(f\"Tensor C:\\n{random_tensor_C}\\n\")\n",
    "print(f\"Tensor D:\\n{random_tensor_D}\\n\")\n",
    "print(f\"Does Tensor C equal Tensor D? (anywhere)\")\n",
    "random_tensor_C == random_tensor_D\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97afe122",
   "metadata": {},
   "source": [
    "## Check GPU availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "93acecf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5d7861fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45c03812",
   "metadata": {},
   "source": [
    "### Putting Tensors on the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "3089b999",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3]) cpu\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3], device='cuda:0')"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create tensor (default on CPU)\n",
    "tensor = torch.tensor([1, 2, 3])\n",
    "\n",
    "# Tensor not on GPU\n",
    "print(tensor, tensor.device)\n",
    "\n",
    "# Move tensor to GPU (if available)\n",
    "tensor_on_gpu = tensor.to(device)\n",
    "tensor_on_gpu"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b56f83a3",
   "metadata": {},
   "source": [
    "### Moving Tensors back to the CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c819fcd",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[79], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# If tensor is on GPU, can't transform it to NumPy (this will error)\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mtensor_on_gpu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnumpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
     ]
    }
   ],
   "source": [
    "# If tensor is on GPU, can't transform it to NumPy (this will error)\n",
    "# tensor_on_gpu.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "1a376197",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Instead, copy the tensor back to cpu\n",
    "tensor_back_on_cpu = tensor_on_gpu.cpu().numpy()\n",
    "tensor_back_on_cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "4529c0e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3], device='cuda:0')"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor_on_gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1186747c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}