tensor_operations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensor Operations with Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manually Create Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n",
      "torch.Size([4, 3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1, 2, 3],\n",
    "                 [4, 5, 6],\n",
    "                 [7, 8, 9],\n",
    "                 [10, 11, 12]])\n",
    "print(x)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Randomly initialized 5x3 tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2888, 0.0274, 0.8206],\n",
      "        [0.6562, 0.7636, 0.7981],\n",
      "        [0.2962, 0.8010, 0.0344],\n",
      "        [0.5068, 0.6048, 0.6641],\n",
      "        [0.0222, 0.9523, 0.0611]])\n",
      "torch.Size([5, 3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand(5, 3)\n",
    "print(x)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5x3 tensor of zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0]])\n",
      "torch.Size([5, 3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.zeros(5, 3, dtype=torch.long)\n",
    "print(x)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Element Access"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([[1, 2, 3],\n",
    "                 [4, 5, 6],\n",
    "                 [7, 8, 9],\n",
    "                 [10, 11, 12]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get single element at 3rd row, 2nd column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[2, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get entire 4th row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 11, 12])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[3, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get entire 3rd column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 3,  6,  9, 12])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:, 2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get all elements greater than 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 6,  7,  8,  9, 10, 11, 12])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[x > 5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensor Arithmetic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    }
   ],
   "source": [
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2,  3,  4],\n",
       "        [ 6,  7,  8],\n",
       "        [10, 11, 12],\n",
       "        [14, 15, 16]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = torch.tensor([[1, 1, 1],\n",
    "                  [2, 2, 2],\n",
    "                  [3, 3, 3],\n",
    "                  [4, 4, 4]])\n",
    "x + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broadcasting\n",
    "Tensor arithmetic between two tensors with different dimensions. Broadcast expands the lower dimensions by replicating the elements on more dimensions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Row wise broadcast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n",
      "tensor([1, 2, 3])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 2,  4,  6],\n",
       "        [ 5,  7,  9],\n",
       "        [ 8, 10, 12],\n",
       "        [11, 13, 15]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = torch.tensor([1, 2, 3])\n",
    "print(x)\n",
    "print(b)\n",
    "x+b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Element wise broadcast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n",
      "5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 6,  7,  8],\n",
       "        [ 9, 10, 11],\n",
       "        [12, 13, 14],\n",
       "        [15, 16, 17]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x)\n",
    "print(5)\n",
    "x+5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sum of all elements in tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(78)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x)\n",
    "torch.sum(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sum of elements row-wise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([ 6, 15, 24, 33])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x)\n",
    "torch.sum(x, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reshaping Tensor to 4x3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  2,  3,  4],\n",
       "        [ 5,  6,  7,  8],\n",
       "        [ 9, 10, 11, 12]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x)\n",
    "x.reshape(3, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transpose tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  4,  7, 10],\n",
       "        [ 2,  5,  8, 11],\n",
       "        [ 3,  6,  9, 12]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x)\n",
    "torch.transpose(x, 0, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with Multiple Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "tensor([[ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([[1, 2, 3],\n",
    "                 [4, 5, 6]])\n",
    "b = torch.tensor([[7, 8, 9],\n",
    "                  [10, 11, 12]])\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Subtract tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 7,  8,  9],\n",
      "        [10, 11, 12]])\n",
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[6, 6, 6],\n",
       "        [6, 6, 6]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(b)\n",
    "print(a)\n",
    "b - a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Element wise matrix multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "tensor([[ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 7, 16, 27],\n",
       "        [40, 55, 72]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(a)\n",
    "print(b)\n",
    "a*b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dot product matrix multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "tensor([[ 7, 10],\n",
      "        [ 8, 11],\n",
      "        [ 9, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 50,  68],\n",
       "        [122, 167]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(a)\n",
    "bt = torch.transpose(b, 0, 1)\n",
    "print(bt)\n",
    "torch.matmul(a, bt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenating two tensors\n",
    "Appends elements of tensor 2 into tensor 1. Requires that elements of tensor 1 have the same dimension as tensor 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "tensor([[ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  2,  3],\n",
       "        [ 4,  5,  6],\n",
       "        [ 7,  8,  9],\n",
       "        [10, 11, 12]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(a)\n",
    "print(b)\n",
    "torch.cat((a, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stacking two tensors\n",
    "Combines tensor 1 and tensor 2 into one tensor. Requires that tensor 1 has the same dimension as tensor 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "tensor([[ 7,  8,  9],\n",
      "        [10, 11, 12]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1,  2,  3],\n",
       "         [ 4,  5,  6]],\n",
       "\n",
       "        [[ 7,  8,  9],\n",
       "         [10, 11, 12]]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(a)\n",
    "print(b)\n",
    "torch.stack((a, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autograd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Automatic Gradient Tracking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 1.],\n",
      "        [1., 1.]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "x = torch.ones(2, 2, requires_grad=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[27., 27.],\n",
      "        [27., 27.]], grad_fn=<MulBackward0>)\n",
      "tensor(27., grad_fn=<MeanBackward0>)\n"
     ]
    }
   ],
   "source": [
    "z = y * y * 3\n",
    "out = z.mean()\n",
    "\n",
    "print(z)\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Backprop Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.5000, 4.5000],\n",
      "        [4.5000, 4.5000]])\n"
     ]
    }
   ],
   "source": [
    "out.backward()\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Speeding up computation with GPU"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}