Colab/Swift for TensorFlow for using PyTorch users No2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Swift for TensorFlow for using PyTorch users No2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "swift",
      "display_name": "Swift"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/khirotaka/Swift4TF/blob/master/Colab/Swift%20for%20TensorFlow%20for%20using%20PyTorch%20users%20No2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-SHf43iFxP4V",
        "colab_type": "text"
      },
      "source": [
        "# 自動微分"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZRlD4utdPuX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import TensorFlow\n",
        "\n",
        "Context.local.randomSeed = TensorFlowSeed(graph: 0, op: 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E38cwYE0xlWj",
        "colab_type": "text"
      },
      "source": [
        "PyTorchでは、以下のように `requires_grad=True` を使って微分するしないを明記するが、\n",
        "```python\n",
        "x = torch.ones(2, 2, requires_grad=True)\n",
        "```\n",
        "\n",
        "Swift for TensorFlow では、TensorFlow 2系 の `tf.GradientTape()` に近い書き方をする。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xs1Lzw67xb82",
        "colab_type": "code",
        "outputId": "f553e978-280e-4378-d8f2-2d4d79cd4a2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "var x = Tensor<Float>(ones: [2, 2])\n",
        "print(x)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1.0, 1.0],\r\n",
            " [1.0, 1.0]]\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UcjtqMypxjXS",
        "colab_type": "code",
        "outputId": "5789605c-e93b-4821-b32d-d0e29065d887",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "var y = TensorFlow.valueWithGradient(at: x) { x in      // PyTorch の .backward()\n",
        "    (x + 2.0).sum()\n",
        "}\n",
        "\n",
        "y       // 二つの要素を持つタプル"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "▿ 2 elements\n",
              "  - value : 12.0\n",
              "  - gradient : [[1.0, 1.0],\n",
              " [1.0, 1.0]]\n"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AYSPb-VR2n6i",
        "colab_type": "text"
      },
      "source": [
        "戻り値は タプルなので、次のように書けば各要素にアクセスできる。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Frq5-pKP2nUm",
        "colab_type": "code",
        "outputId": "5981baf0-de8b-4dd3-b36e-52d795c914be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "print(\"Value: \\(y.0)\")\n",
        "print(\"Gradient: \\n \\(y.1)\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Value: 12.0\r\n",
            "Gradient: \r\n",
            " [[1.0, 1.0],\r\n",
            " [1.0, 1.0]]\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rzUMYclXz9iq",
        "colab_type": "text"
      },
      "source": [
        "## 導関数を求める"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fB0gvpOZzQ7W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@differentiable\n",
        "func f(_ inputs: Double) -> Double {\n",
        "    return inputs * inputs   // x^2\n",
        "}\n",
        "\n",
        "let df = gradient(of: f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mrUEBKp0g3N",
        "colab_type": "code",
        "outputId": "46e0c5ff-7712-40a0-85e4-d8ea6ee951c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "f(5.0)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "25.0\n"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exE-LZKX0kNw",
        "colab_type": "code",
        "outputId": "24a695d9-e8e5-4822-af2e-ea576a1f2731",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df(5.0)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10.0\n"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lj3xr3fgBVp-",
        "colab_type": "text"
      },
      "source": [
        "PyTorchの \n",
        "```python\n",
        "with torch.no_grad():\n",
        "    ...\n",
        "```\n",
        "\n",
        "的なやつ\n",
        "\n",
        "`withoutDerivative(at: )` を使う事で、引数に指定した関数は勾配計算を行わない。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRhAU3As0q6t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "25af0048-2380-43b3-8d56-b4037913f5b3"
      },
      "source": [
        "let a = 10.0\n",
        "\n",
        "gradient(at: a) { a in \n",
        "    f(a) + withoutDerivative(at: f(a))  // 2x\n",
        "}"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "20.0\n"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5zUnGN-8CkLm",
        "colab_type": "text"
      },
      "source": [
        "## 簡単なニューラルネット"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HNZZeoqRFywt",
        "colab_type": "text"
      },
      "source": [
        "Swift for TensorFlowでのモデル定義は、PyTorchのそれとほとんど同じ。  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBLgQAs-FrXG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@differentiable\n",
        "func mish(_ inputs: Tensor<Float>) -> Tensor<Float> {\n",
        "    return inputs * tanh(softplus(inputs))\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9KtAT54tCmT6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "struct Network: Module {\n",
        "    var fc1 = Sequential(\n",
        "        Dense<Float>(inputSize: 4, outputSize: 128, activation: mish),\n",
        "        Dense<Float>(inputSize: 128, outputSize: 64, activation: mish)\n",
        "    )\n",
        "    \n",
        "    var fc2 = Dense<Float>(inputSize: 64, outputSize: 10)\n",
        "\n",
        "    @differentiable\n",
        "    func callAsFunction(_ input: Tensor<Float>) -> Tensor<Float> {\n",
        "        let out = fc1(input)\n",
        "        return mish(fc2(out))\n",
        "    }\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UGs03OQ0DDMY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Context.local.learningPhase = .training    // PyTorch の model.train() と同じ\n",
        "\n",
        "var model = Network()\n",
        "let optimizer = Adam(for: model, learningRate: 0.01)\n",
        "\n",
        "\n",
        "let x = Tensor<Float>(randomNormal: [10, 4])\n",
        "let y = Tensor<Int32>([0, 1, 0, 1, 8, 3, 4, 6, 9, 2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KrRB5ExeDgC1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "2fcf67c6-2a63-4464-c320-2ffd57bd8dba"
      },
      "source": [
        "for _ in 0..<10 {\n",
        "    let delta_model = gradient(at: model) { model -> Tensor<Float> in\n",
        "        let pred = model(x)\n",
        "        let loss = softmaxCrossEntropy(logits: pred, labels: y)\n",
        "        print(loss)\n",
        "        return loss\n",
        "    }\n",
        "    optimizer.update(&model, along: delta_model)\n",
        "}"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.32212\r\n",
            "2.103107\r\n",
            "1.854009\n",
            "1.6049321\n",
            "1.3833803\n",
            "1.1867671\n",
            "1.0061058\n",
            "0.8452984\n",
            "0.7096749\n",
            "0.59515756\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnlZs-mqERx2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}