kal2257_hw3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "H2BVH_JD9WiA"
   },
   "source": [
    "Below we will try and fit a Logisitc Regression Model step by step for the XOR problem.\n",
    "For this model, we have $x_1$ and $x_2$ are either 0/1 each and $y = x_1 + x_2 - 2x_1x_2$. Notice that this is True (1) if $x_1 = 1$ and $x_2 = 0$ OR $x_1 = 0$ and $x_2 = 1$; $y$ is zero otherwise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "wiFGf-9H9X3d"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n"
    
    
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1TRwUp469X-r"
   },
   "outputs": [],
   "source": [
    "x_data = [[0, 0], [0, 1], [1, 0], [1, 1]]\n",
    "y_data = [[0], [1], [1], [0]]\n",
    "x_data = torch.Tensor(x_data)\n",
    "y_data = torch.Tensor(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YhPtW7jO0PSx"
   },
   "outputs": [],
   "source": [
    "# Define each tensor to be 1x1 and have them require a gradient for tracking; these are parameters\n",
    "dim = 1,1\n",
    "beta_1 = torch.rand(dim,requires_grad=True)\n",
    "beta_2 = torch.rand(dim,requires_grad=True)\n",
    "alpha  = torch.rand(dim,requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dJOZvBwgM07g",
    "outputId": "7db56d66-1774-45d4-c26c-ecdd965adb6a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "Loss: 0.7420799136161804 Accuracy: 0.5\n",
      "Epoch: 1\n",
      "Loss: 0.7406654953956604 Accuracy: 0.5\n",
      "Epoch: 2\n",
      "Loss: 0.7392904162406921 Accuracy: 0.5\n",
      "Epoch: 3\n",
      "Loss: 0.7379539012908936 Accuracy: 0.5\n",
      "Epoch: 4\n",
      "Loss: 0.7366548776626587 Accuracy: 0.5\n",
      "Epoch: 5\n",
      "Loss: 0.7353924512863159 Accuracy: 0.5\n",
      "Epoch: 6\n",
      "Loss: 0.734165608882904 Accuracy: 0.5\n",
      "Epoch: 7\n",
      "Loss: 0.7329734563827515 Accuracy: 0.5\n",
      "Epoch: 8\n",
      "Loss: 0.7318153381347656 Accuracy: 0.5\n",
      "Epoch: 9\n",
      "Loss: 0.730690062046051 Accuracy: 0.5\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import grad\n",
    "lr = 0.01\n",
    "\n",
    "for epoch in range(10):\n",
    "  for x, y in zip(x_data, y_data):\n",
    "\n",
    "    # Have z be beta_2*x[0] + beta_1*x[1] + alpha\n",
    "    z = beta_2*x[0] + beta_1*x[1] + alpha\n",
    "\n",
    "    # Push z through a nn.Sigmoid layer to get the p(y=1)\n",
    "    a = torch.sigmoid(z)\n",
    "\n",
    "    # Write the loss manually between y and a\n",
    "    loss = -1*((y*torch.log(a)) + ((1-y)*torch.log(1-a)))\n",
    "\n",
    "    # Get the loss gradients; the gradients with respect to alpha, beta_1, beta_2\n",
    "    loss.backward()\n",
    "    grad_beta2 = beta_2.grad\n",
    "    grad_beta1 = beta_1.grad\n",
    "    grad_alpha = alpha.grad\n",
    "\n",
    "    # Manually update the gradients\n",
    "    # What we do below is wrapped within this clause because weights have required_grad=True but we don't need to track this in autograd\n",
    "    with torch.no_grad():\n",
    "        # Do an update for each parameter\n",
    "          beta_2 = beta_2 - lr*grad_beta2\n",
    "          beta_1 = beta_1 - lr*grad_beta1\n",
    "          alpha  = alpha  - lr*grad_alpha\n",
    "\n",
    "        # Manually zero the gradients after updating weights\n",
    "          beta_2.grad = None\n",
    "          beta_1.grad = None\n",
    "          alpha.grad  = None\n",
    "          beta_2.requires_grad = True\n",
    "          beta_1.requires_grad = True\n",
    "          alpha.requires_grad = True\n",
    "\n",
    "  # Manually get the accuracy of the model after each epoch\n",
    "  with torch.no_grad():\n",
    "    print(f'Epoch: {epoch}')\n",
    "    y_pred = []\n",
    "    loss = 0.0\n",
    "\n",
    "    for x, y in zip(x_data, y_data):\n",
    "      # Get z\n",
    "      z = beta_2*x[0] + beta_1*x[1] + alpha\n",
    "\n",
    "      # Get a\n",
    "      a = torch.sigmoid(z)\n",
    "\n",
    "\n",
    "      # Get the loss\n",
    "      loss += -1*((y*torch.log(a)) + ((1-y)*torch.log(1-a)))\n",
    "\n",
    "      # Get the prediction given a\n",
    "      y_pred =y_pred +[1 if a >= 0.5 else 0]\n",
    "\n",
    "    #Get the current accuracy over 4 points; make this a tensor\n",
    "    y_pred=torch.FloatTensor(y_pred)\n",
    "\n",
    "    accuracy =torch.sum(torch.eq(y_pred,y_data.T))/4\n",
    "    loss = loss / 4\n",
    "\n",
    "    #Print the accuracy and the loss\n",
    "    #You want the item in the tensor thats 1x1\n",
    "    print('Loss: {} Accuracy: {}'.format(loss.item() ,accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Iojtw_rFAjhY"
   },
   "source": [
    "Exercise 1: Create a 2D tensor and then add a dimension of size 1 inserted at the 0th axis.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fgdImVPpAm6d",
    "outputId": "0c4ef9c6-3864-4f07-c24d-e2b9c40b4c20"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0598, 0.1709],\n",
       "         [0.5572, 0.6326]]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim = 2,2\n",
    "x = torch.rand(dim).unsqueeze(0)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A0yfuo7fAneJ"
   },
   "source": [
    "Exercise 2: Remove the extra dimension you just added to the previous tensor.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "goe1-DBRAnnj",
    "outputId": "19b9bb9e-67d7-44cb-bb0a-c68aa8a97f80"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0598, 0.1709],\n",
       "        [0.5572, 0.6326]])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = x.squeeze(0)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oAhtAtk5Any4"
   },
   "source": [
    "Exercise 3: Create a random tensor of shape 5x3 in the interval [3, 7)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZCcFowEjAn8w",
    "outputId": "82aee182-e444-4777-ba4d-1a28d2ca36e3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.8692, 4.2555, 3.6053],\n",
       "        [3.3966, 5.2864, 5.9531],\n",
       "        [6.2919, 5.4445, 5.0535],\n",
       "        [3.3911, 6.2098, 6.6665],\n",
       "        [6.5670, 5.6480, 4.0700]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim = 5,3\n",
    "start, end = 3,7\n",
    "diff = end - start\n",
    "x = torch.rand(dim)\n",
    "x = start + diff*x\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PvNprVRlAoEC"
   },
   "source": [
    "Exercise 4: Create a tensor with values from a normal distribution (mean=0, std=1).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Dgirc4kGAoKa",
    "outputId": "3528dbae-329b-4614-8e2f-f92cfafa0ddb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8529,  0.0502,  0.5259],\n",
       "        [-0.7046,  0.3235, -0.2141],\n",
       "        [ 1.4503,  0.9467, -0.0790],\n",
       "        [-0.3112,  1.3725,  0.4459],\n",
       "        [ 0.6614,  0.1794,  0.4289]])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim = 5,3\n",
    "mu,sigma = 0,1\n",
    "x = torch.normal(mu,sigma, size=dim)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i1nIIGp8AoQL"
   },
   "source": [
    "exercise 5: Retrieve the indexes of all the non zero elements in the tensor torch.Tensor([1, 1, 1, 0, 1]).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sCv5zbq3AoV-",
    "outputId": "9942ce9c-c763-49e0-93c2-b6baf7d81333"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0],\n",
       "        [1],\n",
       "        [2],\n",
       "        [4]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.Tensor([1, 1, 1, 0, 1]).nonzero()\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ckErX5U1Aocz"
   },
   "source": [
    "Exercise 6: Create a random tensor of size (3,1) and then horizonally stack 4 copies together.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9D3XYAnoAoig",
    "outputId": "2554e44d-f170-4b09-e966-1e25b54b4322"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2509, 0.2509, 0.2509, 0.2509],\n",
       "        [0.1755, 0.1755, 0.1755, 0.1755],\n",
       "        [0.7913, 0.7913, 0.7913, 0.7913]])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim = 3,1\n",
    "desired_dim = 4\n",
    "x = torch.rand(dim).expand(dim[0] ,desired_dim)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wKV3ChJrAopD"
   },
   "source": [
    "Exercise 7: Return the batch matrix-matrix product of two 3 dimensional matrices (a=torch.rand(3,4,5), b=torch.rand(3,5,4)).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ge6IErGdAovX",
    "outputId": "dd39067a-72d2-42e8-b955-7aa783e626ee"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1.4494, 1.0602, 1.0511, 1.0479],\n",
       "         [1.3060, 1.1231, 1.5504, 1.6478],\n",
       "         [2.6341, 1.7840, 2.8105, 2.8984],\n",
       "         [1.3131, 0.7990, 1.5453, 1.7199]],\n",
       "\n",
       "        [[0.9078, 0.8409, 1.5928, 1.2497],\n",
       "         [1.7301, 1.3984, 2.2090, 1.5277],\n",
       "         [0.4239, 0.4361, 0.7011, 0.3213],\n",
       "         [1.5538, 1.1826, 1.9572, 0.9656]],\n",
       "\n",
       "        [[0.7172, 0.8004, 1.0903, 0.8760],\n",
       "         [0.7382, 1.1985, 1.2859, 1.3656],\n",
       "         [0.6102, 1.1650, 0.7028, 1.0886],\n",
       "         [0.3383, 1.0531, 0.7312, 1.1462]]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim_a = 3,4,5\n",
    "dim_b = 3,5,4\n",
    "a = torch.rand(dim_a)\n",
    "b = torch.rand(dim_b)\n",
    "x = torch.bmm(a, b)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qVI_LI_PA_2e"
   },
   "source": [
    "Exercise 8: Return the batch matrix-matrix product of a 3D matrix and a 2D matrix (a=torch.rand(3,4,5), b=torch.rand(5,4)).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HHv60VM1M07j",
    "outputId": "e57b0461-7427-4b80-a246-3436aa3f0377",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1.5072, 0.9352, 1.9807, 1.7094],\n",
       "         [1.1816, 0.9497, 2.0066, 1.6808],\n",
       "         [1.1476, 0.6570, 1.8228, 1.2792],\n",
       "         [1.1831, 1.0358, 2.1361, 2.0779]],\n",
       "\n",
       "        [[1.2094, 0.8068, 2.0547, 1.4848],\n",
       "         [1.4514, 0.8914, 2.2731, 1.6378],\n",
       "         [0.7146, 0.6993, 1.4329, 1.3858],\n",
       "         [1.3151, 0.8394, 1.9992, 1.6389]],\n",
       "\n",
       "        [[1.2578, 0.7881, 1.6333, 1.6176],\n",
       "         [1.2229, 0.9364, 1.7324, 1.5681],\n",
       "         [1.3912, 0.6937, 1.9344, 1.3331],\n",
       "         [1.5092, 1.2223, 2.6781, 2.1639]]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim_a = 3,4,5\n",
    "dim_b = 5,4\n",
    "a = torch.rand(dim_a)\n",
    "b = torch.rand(dim_b)\n",
    "b_exp = b.unsqueeze(0)\n",
    "desired_dim = [dim_a[0]]+list(b_exp.shape)\n",
    "b_exp = b_exp.expand(desired_dim).squeeze(1)\n",
    "x = torch.bmm(a,b_exp)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KW6NxQIeBAJA"
   },
   "source": [
    "Exercise 9: Create a 1x1 random tensor and get the value inside of this tensor as a scalar. No tensor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o_OFj9hEBAPO",
    "outputId": "32349d97-a959-419a-aa3d-1da4e29e65ff"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8007276058197021"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim = 1,1\n",
    "x = torch.rand(dim).item()\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5_zAwiqrBAVd"
   },
   "source": [
    "Exercise 10: Create a 2x1 tensor and have it require a gradient. Have $x$, this tensor, hold [-2, 1]. Set $y=x_1^2 + x_2^2$ and get the gradient of y wirht respect to $x_1$ and then $x_2$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "z98hDPfEBAcv",
    "outputId": "0cf3b253-2712-40ac-f834-9030a2f24ff8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d/dx1: -4.0 |  d/dx2: 2.0\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[-2.0],[1.0]], requires_grad=True)\n",
    "y = x[0]**2 + x[1]**2\n",
    "y.backward()\n",
    "print('d/dx1:',x.grad[0].item(),'|  d/dx2:',x.grad[1].item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sGfmkpF3BAjy"
   },
   "source": [
    "Exercise 11: Check if cuda is available (it shuld be if in the Runtime setting for colab you choose the GPU). If it is, move $x$ above to a CUDA device. Create a new tensor of the same shape as $x$ and put it on the cpu. Try and add these tensors. What happens. How do you fix this?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2M_Suz2XBAsX",
    "outputId": "590c66bb-53c1-4049-f4e5-293981fa8a87"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 211
    },
    "id": "Lb7LB-vXzEbT",
    "outputId": "263ea551-3e9c-489f-9c67-39e03ca4e626"
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-c2ffd2c3ffef>\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mx2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcpu_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mx1\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mx2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!"
     ]
    }
   ],
   "source": [
    "gpu_device = torch.device(\"cuda\")\n",
    "x1 = x.to(gpu_device)\n",
    "\n",
    "cpu_device = torch.device(\"cpu\")\n",
    "x2 = torch.tensor([[4.0],[-1.0]], requires_grad=True)\n",
    "x2 = x2.to(cpu_device)\n",
    "\n",
    "x1+x2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bdhkTfrd240y"
   },
   "source": [
    "## Answer:\n",
    "The problem is that the tensors are on different devices, and all variables needed for the single computation must be on the same device. This causes an error and the computation cannot be performed. We could fix this by moving both variables x1 and x2 to the same device, either both on the GPU or both on the  CPU."
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}