deep_04.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  np.reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "a = np.array([1,2,3,4])\n",
    "print(a.shape)\n",
    "print(a.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "a = np.array([1,2,3])\n",
    "print(a.shape)\n",
    "print(a.reshape(3,1))\n",
    "print(a.reshape(-1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### axis=0, axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "a = np.arange(8).reshape(2,4)\n",
    "print(a)\n",
    "print(np.sum(a, axis=1))\n",
    "print(np.sum(a, axis=1).reshape(-1,1))\n",
    "print(np.sum(a, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SingleLayer:    \n",
    "    def __init__(self, learning_rate=0.1, l1=0, l2=0):\n",
    "        self.w = None              # 가중치\n",
    "        self.b = None              # 절편\n",
    "        self.losses = []           # 훈련 손실\n",
    "        self.val_losses = []       # 검증 손실\n",
    "        self.w_history = []        # 가중치 기록\n",
    "        self.lr = learning_rate    # 학습률\n",
    "        self.l1 = l1               # L1 손실 하이퍼파라미터\n",
    "        self.l2 = l2               # L2 손실 하이퍼파라미터\n",
    "\n",
    "    def forpass(self, x):\n",
    "        z = np.dot(x, self.w) + self.b        # 선형 출력을 계산합니다.\n",
    "        return z\n",
    "\n",
    "    def backprop(self, x, err):\n",
    "        m = len(x)\n",
    "        w_grad = np.dot(x.T, err) / m         # 가중치에 대한 그래디언트를 계산합니다.\n",
    "        b_grad = np.sum(err) / m              # 절편에 대한 그래디언트를 계산합니다.\n",
    "        return w_grad, b_grad\n",
    "\n",
    "    def activation(self, z):\n",
    "        z = np.clip(z, -100, None)            # 안전한 np.exp() 계산을 위해\n",
    "        a = 1 / (1 + np.exp(-z))              # 시그모이드 계산\n",
    "        return a\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        y = y.reshape(-1, 1)                  # 타깃을 열 벡터로 바꿉니다.\n",
    "        y_val = y_val.reshape(-1, 1)\n",
    "        m = len(x)                            # 샘플 개수를 저장합니다.\n",
    "        self.w = np.ones((x.shape[1], 1))     # 가중치를 초기화합니다.\n",
    "        self.b = 0                            # 절편을 초기화합니다.\n",
    "        self.w_history.append(self.w.copy())  # 가중치를 기록합니다.\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            z = self.forpass(x)               # 정방향 계산을 수행합니다.\n",
    "            a = self.activation(z)            # 활성화 함수를 적용합니다.\n",
    "            err = a - y                   # 오차를 계산합니다.\n",
    "            # 오차를 역전파하여 그래디언트를 계산합니다.\n",
    "            w_grad, b_grad = self.backprop(x, err)\n",
    "            # 그래디언트에서 페널티 항의 미분 값을 더합니다.\n",
    "            w_grad += (self.l1 * np.sign(self.w) + self.l2 * self.w) / m\n",
    "            # 가중치와 절편을 업데이트합니다.\n",
    "            self.w -= self.lr * w_grad\n",
    "            self.b -= self.lr * b_grad\n",
    "            # 가중치를 기록합니다.\n",
    "            self.w_history.append(self.w.copy())\n",
    "            # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "            a = np.clip(a, 1e-10, 1-1e-10)\n",
    "            # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "            loss = np.sum(-(y*np.log(a) + (1-y)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / m)\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "    \n",
    "    def predict(self, x):\n",
    "        z = self.forpass(x)      # 정방향 계산을 수행합니다.\n",
    "        return z > 0             # 스텝 함수를 적용합니다.\n",
    "    \n",
    "    def score(self, x, y):\n",
    "        # 예측과 타깃 열 벡터를 비교하여 True의 비율을 반환합니다.\n",
    "        return np.mean(self.predict(x) == y.reshape(-1, 1))\n",
    "    \n",
    "    def reg_loss(self):\n",
    "        # 가중치에 규제를 적용합니다.\n",
    "        return self.l1 * np.sum(np.abs(self.w)) + self.l2 / 2 * np.sum(self.w**2)\n",
    "    \n",
    "    def update_val_loss(self, x_val, y_val):\n",
    "        z = self.forpass(x_val)            # 정방향 계산을 수행합니다.\n",
    "        a = self.activation(z)             # 활성화 함수를 적용합니다.\n",
    "        a = np.clip(a, 1e-10, 1-1e-10)     # 출력 값을 클리핑합니다.\n",
    "        # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "        val_loss = np.sum(-(y_val*np.log(a) + (1-y_val)*np.log(1-a)))\n",
    "        self.val_losses.append((val_loss + self.reg_loss()) / len(y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DualLayer(SingleLayer):\n",
    "    \n",
    "    def __init__(self, units=10, learning_rate=0.1, l1=0, l2=0):\n",
    "        self.units = units         # 은닉층의 뉴런 개수\n",
    "        self.w1 = None             # 은닉층의 가중치\n",
    "        self.b1 = None             # 은닉층의 절편\n",
    "        self.w2 = None             # 출력층의 가중치\n",
    "        self.b2 = None             # 출력층의 절편\n",
    "        self.a1 = None             # 은닉층의 활성화 출력\n",
    "        self.losses = []           # 훈련 손실\n",
    "        self.val_losses = []       # 검증 손실\n",
    "        self.lr = learning_rate    # 학습률\n",
    "        self.l1 = l1               # L1 손실 하이퍼파라미터\n",
    "        self.l2 = l2               # L2 손실 하이퍼파라미터\n",
    "\n",
    "    def forpass(self, x):\n",
    "        z1 = np.dot(x, self.w1) + self.b1        # 첫 번째 층의 선형 식을 계산합니다\n",
    "        self.a1 = self.activation(z1)            # 활성화 함수를 적용합니다\n",
    "        z2 = np.dot(self.a1, self.w2) + self.b2  # 두 번째 층의 선형 식을 계산합니다.\n",
    "        return z2\n",
    "\n",
    "    def backprop(self, x, err):\n",
    "        m = len(x)       # 샘플 개수\n",
    "        # 출력층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w2_grad = np.dot(self.a1.T, err) / m\n",
    "        b2_grad = np.sum(err, axis=0) / m\n",
    "        # 시그모이드 함수까지 그래디언트를 계산합니다.\n",
    "        err_to_hidden = np.dot(err, self.w2.T) * self.a1 * (1 - self.a1)\n",
    "        # 은닉층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w1_grad = np.dot(x.T, err_to_hidden) / m\n",
    "        b1_grad = np.sum(err_to_hidden, axis=0) / m\n",
    "        return w1_grad, b1_grad, w2_grad, b2_grad\n",
    "\n",
    "    def init_weights(self, n_features):\n",
    "        self.w1 = np.ones((n_features, self.units))  # (특성 개수, 은닉층의 크기)\n",
    "        self.b1 = np.zeros(self.units)               # 은닉층의 크기\n",
    "        self.w2 = np.ones((self.units, 1))           # (은닉층의 크기, 1)\n",
    "        self.b2 = 0\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        y = y.reshape(-1, 1)          # 타깃을 열 벡터로 바꿉니다.\n",
    "        y_val = y_val.reshape(-1, 1)\n",
    "        m = len(x)                    # 샘플 개수를 저장합니다.\n",
    "        self.init_weights(x.shape[1]) # 은닉층과 출력층의 가중치를 초기화합니다.\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            a = self.training(x, y, m)\n",
    "            # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "            a = np.clip(a, 1e-10, 1-1e-10)\n",
    "            # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "            loss = np.sum(-(y*np.log(a) + (1-y)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / m)\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "            \n",
    "    def training(self, x, y, m):\n",
    "        z = self.forpass(x)       # 정방향 계산을 수행합니다.\n",
    "        a = self.activation(z)    # 활성화 함수를 적용합니다.\n",
    "        err = (a - y)            # 오차를 계산합니다.\n",
    "        # 오차를 역전파하여 그래디언트를 계산합니다.\n",
    "        w1_grad, b1_grad, w2_grad, b2_grad = self.backprop(x, err)\n",
    "        # 그래디언트에서 페널티 항의 미분 값을 뺍니다\n",
    "        w1_grad += (self.l1 * np.sign(self.w1) + self.l2 * self.w1) / m\n",
    "        w2_grad += (self.l1 * np.sign(self.w2) + self.l2 * self.w2) / m\n",
    "        # 은닉층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w1 -= self.lr * w1_grad\n",
    "        self.b1 -= self.lr * b1_grad\n",
    "        # 출력층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w2 -= self.lr * w2_grad\n",
    "        self.b2 -= self.lr * b2_grad\n",
    "        return a\n",
    "    \n",
    "    def reg_loss(self):\n",
    "        # 은닉층과 출력층의 가중치에 규제를 적용합니다.\n",
    "        return self.l1 * (np.sum(np.abs(self.w1)) + np.sum(np.abs(self.w2))) + \\\n",
    "               self.l2 / 2 * (np.sum(self.w1**2) + np.sum(self.w2**2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomInitNetwork(DualLayer):\n",
    "    \n",
    "    def init_weights(self, n_features):\n",
    "        np.random.seed(42)\n",
    "        self.w1 = np.random.normal(0, 1, \n",
    "                                   (n_features, self.units))  # (특성 개수, 은닉층의 크기)\n",
    "        self.b1 = np.zeros(self.units)                        # 은닉층의 크기\n",
    "        self.w2 = np.random.normal(0, 1, \n",
    "                                   (self.units, 1))           # (은닉층의 크기, 1)\n",
    "        self.b2 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MinibatchNetwork(RandomInitNetwork):\n",
    "    \n",
    "    def __init__(self, units=10, batch_size=32, learning_rate=0.1, l1=0, l2=0):\n",
    "        super().__init__(units, learning_rate, l1, l2)\n",
    "        self.batch_size = batch_size     # 배치 크기\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        y_val = y_val.reshape(-1, 1)     # 타깃을 열 벡터로 바꿉니다.\n",
    "        self.init_weights(x.shape[1])    # 은닉층과 출력층의 가중치를 초기화합니다.\n",
    "        np.random.seed(42)\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            loss = 0\n",
    "            # 제너레이터 함수에서 반환한 미니배치를 순환합니다.\n",
    "            for x_batch, y_batch in self.gen_batch(x, y):\n",
    "                y_batch = y_batch.reshape(-1, 1) # 타깃을 열 벡터로 바꿉니다.\n",
    "                m = len(x_batch)                 # 샘플 개수를 저장합니다.\n",
    "                a = self.training(x_batch, y_batch, m)\n",
    "                # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "                a = np.clip(a, 1e-10, 1-1e-10)\n",
    "                # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "                loss += np.sum(-(y_batch*np.log(a) + (1-y_batch)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / len(x))\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "\n",
    "    # 미니배치 제너레이터 함수\n",
    "    def gen_batch(self, x, y):\n",
    "        length = len(x)\n",
    "        bins = length // self.batch_size # 미니배치 횟수\n",
    "        if length % self.batch_size:\n",
    "            bins += 1                    # 나누어 떨어지지 않을 때\n",
    "        indexes = np.random.permutation(np.arange(len(x))) # 인덱스를 섞습니다.\n",
    "        x = x[indexes]\n",
    "        y = y[indexes]\n",
    "        for i in range(bins):\n",
    "            start = self.batch_size * i\n",
    "            end = self.batch_size * (i + 1)\n",
    "            yield x[start:end], y[start:end]   # batch_size만큼 슬라이싱하여 반환합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([4,3,5,6,2,7,1])\n",
    "y = np.array([0,0,0,0,0,1,0])\n",
    "print(np.max(a))\n",
    "print(np.argmax(a))\n",
    "print(np.argmax(a) == np.argmax(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([[4,3,5,6,2,7,1],\n",
    "              [3,4,7,5,6,2,1]])\n",
    "y = np.array([[0,0,0,0,0,1,0],\n",
    "              [0,0,1,0,0,0,0]])\n",
    "print(np.argmax(a, axis=1))\n",
    "print( np.mean(np.argmax(a, axis=1) == np.argmax(y, axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiClassNetwork(MinibatchNetwork):\n",
    "    def __init__(self, units=10, batch_size=32, learning_rate=0.1, l1=0, l2=0):\n",
    "        self.units = units         # 은닉층의 뉴런 개수\n",
    "        self.batch_size = batch_size     # 배치 크기\n",
    "        self.w1 = None             # 은닉층의 가중치\n",
    "        self.b1 = None             # 은닉층의 절편\n",
    "        self.w2 = None             # 출력층의 가중치\n",
    "        self.b2 = None             # 출력층의 절편\n",
    "        self.a1 = None             # 은닉층의 활성화 출력\n",
    "        self.losses = []           # 훈련 손실\n",
    "        self.val_losses = []       # 검증 손실\n",
    "        self.lr = learning_rate    # 학습률\n",
    "        self.l1 = l1               # L1 손실 하이퍼파라미터\n",
    "        self.l2 = l2               # L2 손실 하이퍼파라미터\n",
    "\n",
    "    def forpass(self, x):\n",
    "        z1 = np.dot(x, self.w1) + self.b1        # 첫 번째 층의 선형 식을 계산합니다\n",
    "        self.a1 = self.sigmoid(z1)               # 활성화 함수를 적용합니다\n",
    "        z2 = np.dot(self.a1, self.w2) + self.b2  # 두 번째 층의 선형 식을 계산합니다.\n",
    "        return z2\n",
    "\n",
    "    def backprop(self, x, err):\n",
    "        m = len(x)       # 샘플 개수\n",
    "        # 출력층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w2_grad = np.dot(self.a1.T, err) / m\n",
    "        b2_grad = np.sum(err) / m\n",
    "        # 시그모이드 함수까지 그래디언트를 계산합니다.\n",
    "        err_to_hidden = np.dot(err, self.w2.T) * self.a1 * (1 - self.a1)\n",
    "        # 은닉층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w1_grad = np.dot(x.T, err_to_hidden) / m\n",
    "        b1_grad = np.sum(err_to_hidden, axis=0) / m\n",
    "        return w1_grad, b1_grad, w2_grad, b2_grad\n",
    "    \n",
    "    def sigmoid(self, z):\n",
    "        z = np.clip(z, -100, None)            # 안전한 np.exp() 계산을 위해\n",
    "        a = 1 / (1 + np.exp(-z))              # 시그모이드 계산\n",
    "        return a\n",
    "    \n",
    "    def softmax(self, z):\n",
    "        # 소프트맥스 함수\n",
    "        z = np.clip(z, -100, None)            # 안전한 np.exp() 계산을 위해\n",
    "        exp_z = np.exp(z)\n",
    "        return exp_z / np.sum(exp_z, axis=1).reshape(-1, 1)\n",
    " \n",
    "    def init_weights(self, n_features, n_classes):\n",
    "        self.w1 = np.random.normal(0, 1, \n",
    "                                   (n_features, self.units))  # (특성 개수, 은닉층의 크기)\n",
    "        self.b1 = np.zeros(self.units)                        # 은닉층의 크기\n",
    "        self.w2 = np.random.normal(0, 1, \n",
    "                                   (self.units, n_classes))   # (은닉층의 크기, 클래스 개수)\n",
    "        self.b2 = np.zeros(n_classes)\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        np.random.seed(42)\n",
    "        self.init_weights(x.shape[1], y.shape[1])    # 은닉층과 출력층의 가중치를 초기화합니다.\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            loss = 0\n",
    "            print('.', end='')\n",
    "            # 제너레이터 함수에서 반환한 미니배치를 순환합니다.\n",
    "            for x_batch, y_batch in self.gen_batch(x, y):\n",
    "                a = self.training(x_batch, y_batch)\n",
    "                # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "                a = np.clip(a, 1e-10, 1-1e-10)\n",
    "                # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "                loss += np.sum(-y_batch*np.log(a))\n",
    "            self.losses.append((loss + self.reg_loss()) / len(x))\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "\n",
    "    # 미니배치 제너레이터 함수\n",
    "    def gen_batch(self, x, y):\n",
    "        length = len(x)\n",
    "        bins = length // self.batch_size # 미니배치 횟수\n",
    "        if length % self.batch_size:\n",
    "            bins += 1                    # 나누어 떨어지지 않을 때\n",
    "        indexes = np.random.permutation(np.arange(len(x))) # 인덱스를 섞습니다.\n",
    "        x = x[indexes]\n",
    "        y = y[indexes]\n",
    "        for i in range(bins):\n",
    "            start = self.batch_size * i\n",
    "            end = self.batch_size * (i + 1)\n",
    "            yield x[start:end], y[start:end]   # batch_size만큼 슬라이싱하여 반환합니다.\n",
    "            \n",
    "    def training(self, x, y):\n",
    "        m = len(x)                # 샘플 개수를 저장합니다.\n",
    "        z = self.forpass(x)       # 정방향 계산을 수행합니다.\n",
    "        a = self.softmax(z)       # 활성화 함수를 적용합니다.\n",
    "        err = (a - y)            # 오차를 계산합니다.\n",
    "        # 오차를 역전파하여 그래디언트를 계산합니다.\n",
    "        w1_grad, b1_grad, w2_grad, b2_grad = self.backprop(x, err)\n",
    "        # 그래디언트에서 페널티 항의 미분 값을 뺍니다\n",
    "        w1_grad += (self.l1 * np.sign(self.w1) + self.l2 * self.w1) / m\n",
    "        w2_grad += (self.l1 * np.sign(self.w2) + self.l2 * self.w2) / m\n",
    "        # 은닉층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w1 -= self.lr * w1_grad\n",
    "        self.b1 -= self.lr * b1_grad\n",
    "        # 출력층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w2 -= self.lr * w2_grad\n",
    "        self.b2 -= self.lr * b2_grad\n",
    "        return a\n",
    "   \n",
    "    def predict(self, x):\n",
    "        z = self.forpass(x)          # 정방향 계산을 수행합니다.\n",
    "        return np.argmax(z, axis=1)  # 가장 큰 값의 인덱스를 반환합니다.\n",
    "    \n",
    "    def score(self, x, y):\n",
    "        # 예측과 타깃 열 벡터를 비교하여 True의 비율을 반환합니다.\n",
    "        return np.mean(self.predict(x) == np.argmax(y, axis=1))\n",
    "\n",
    "    def reg_loss(self):\n",
    "        # 은닉층과 출력층의 가중치에 규제를 적용합니다.\n",
    "        return self.l1 * (np.sum(np.abs(self.w1)) + np.sum(np.abs(self.w2))) + \\\n",
    "               self.l2 / 2 * (np.sum(self.w1**2) + np.sum(self.w2**2))\n",
    "\n",
    "    def update_val_loss(self, x_val, y_val):\n",
    "        z = self.forpass(x_val)            # 정방향 계산을 수행합니다.\n",
    "        a = self.softmax(z)                # 활성화 함수를 적용합니다.\n",
    "        a = np.clip(a, 1e-10, 1-1e-10)     # 출력 값을 클리핑합니다.\n",
    "        # 크로스 엔트로피 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "        val_loss = np.sum(-y_val*np.log(a))\n",
    "        self.val_losses.append((val_loss + self.reg_loss()) / len(y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# import keras\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "(x_train_all, y_train_all), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x_train_all.shape, y_train_all.shape)\n",
    "print(x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(x_train_all[0], cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y_train_all[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = ['티셔츠/윗도리', '바지', '스웨터', '드레스', '코트', \n",
    "               '샌들', '셔츠', '스니커즈', '가방', '앵클부츠']\n",
    "print(class_names[y_train_all[3]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### np.bincount 예제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([0,4,4,1,2,3,3,3,3,4,5,0])  # (12,)\n",
    "print(len(a))\n",
    "print(a.shape[0])\n",
    "print(np.bincount(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y_train_all))\n",
    "np.bincount(y_train_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y_test))\n",
    "np.bincount(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train_all, y_train_all, stratify=y_train_all, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y_train))\n",
    "np.bincount(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y_val))\n",
    "np.bincount(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x_train[0].shape)\n",
    "print(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train / 255\n",
    "x_val = x_val / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(-1, 784)\n",
    "x_val = x_val.reshape(-1, 784)\n",
    "print(x_train.shape, x_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(x_train[0].reshape(28,28), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.utils.to_categorical([0, 1, 3]) # 원 핫 인코딩"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_encoded = tf.keras.utils.to_categorical(y_train)\n",
    "y_val_encoded = tf.keras.utils.to_categorical(y_val)\n",
    "print(y_train_encoded.shape, y_val_encoded.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y_train[0], y_train_encoded[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc = MultiClassNetwork(units=100, batch_size=256) # x_train_batch.shape = (256,784) w1.shape=(784,100)\n",
    "                                                  # b1.shape = (100,)\n",
    "                                                  # a1.shape = (256,100) , w2.shape = (100,10)\n",
    "                                                  # b2.shape = (10,) , a2.shape(256,10)\n",
    "fc.fit(x_train, y_train_encoded, \n",
    "       x_val=x_val, y_val=y_val_encoded, epochs=40)\n",
    "plt.plot(fc.losses)\n",
    "plt.plot(fc.val_losses)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('iteration')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 테스트 데이터로 평가하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_s = x_test/255\n",
    "x_test_r = x_test_s.reshape(-1,784)\n",
    "y_test_encoded = tf.keras.utils.to_categorical(y_test)\n",
    "print(x_test_r.shape)\n",
    "print(y_test_encoded.shape)\n",
    "score = fc.score(x_test_r, y_test_encoded)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 다중 분류 케라스로 구현하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "model = Sequential()\n",
    "model.add(Dense(100, activation='sigmoid', input_shape=(784,)))  #   (256, 784)(784, 100) + (100,) => (256, 100)\n",
    "model.add(Dense(10, activation='softmax'))                       #   (256, 100)(100, 10) + (10,) => (256, 10)\n",
    "model.compile(optimizer='sgd', loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(x_train, y_train_encoded, epochs=40, \n",
    "                    validation_data=(x_val, y_val_encoded))\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "x = np.array([2, 8, 3, 7, 1, 2, 0, 4, 5])\n",
    "w = np.array([2, 1, 5, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_r = np.flip(w)\n",
    "\n",
    "print( w[::-1] )\n",
    "print(w_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(6):\n",
    "    print(np.dot(x[i:i+4], w_r.reshape(-1,1)), end=' ')\n",
    "      #  (4,)(4,1) => (1,)\n",
    "\n",
    "#     print(np.dot(x[i:i+4], w_r), end=' ')\n",
    "#       #        (4,) (4,) => ()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import convolve\n",
    "convolve(x, w, mode='valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import correlate\n",
    "correlate(x, w, mode='valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import correlate\n",
    "correlate(x, w, mode='full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import correlate\n",
    "correlate(x, w, mode='same')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "x = np.array([[1, 2, 3],\n",
    "              [4, 5, 6],\n",
    "              [7, 8, 9]])\n",
    "w = np.array([[2, 0], \n",
    "              [0, 0]])\n",
    "from scipy.signal import correlate2d\n",
    "cout = correlate2d(x, w, mode='valid')\n",
    "print(cout.shape)\n",
    "print(cout)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([[1, 2, 3],\n",
    "              [4, 5, 6],\n",
    "              [7, 8, 9]])\n",
    "w = np.array([[2, 0], [0, 0]])\n",
    "from scipy.signal import correlate2d\n",
    "correlate2d(x, w, mode='same')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 랠루 활성화 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.arange(-5,5,0.1)\n",
    "y = np.maximum(0,x) \n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 합성곱 연산 ( image 개수 : 1, 채널수: 1, 필터수: 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.layers import *\n",
    "import matplotlib.pyplot as plt\n",
    "image = tf.constant([[[[1],[2],[3]],\n",
    "                      [[4],[5],[6]],\n",
    "                      [[7],[8],[9]]]], dtype=np.float32)\n",
    "# print(type(image))\n",
    "print(image.shape)\n",
    "# print(image)\n",
    "# print(image.numpy())\n",
    "print(image.numpy().reshape(3,3).shape)\n",
    "plt.imshow(image.numpy().reshape(3,3), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = np.array([  [[[1.]], [[1.]]]  ,  [[[1.]],[[1.]]]   ])\n",
    "print(\"weight.shape=\", weight.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 호출 가능한 객체"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Foo:\n",
    "    def __call__(self, data):\n",
    "        print(\"Foo.__call__(%d)\"%data)\n",
    "        \n",
    "Foo()(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "weight_init = tf.constant_initializer(weight)\n",
    "print(type(weight_init))\n",
    "conv2d = tf.keras.layers.Conv2D(filters=1, kernel_size=2, padding='valid', kernel_initializer=weight_init)(image)\n",
    "print(\"conv2d.shape\", conv2d.shape)\n",
    "print(conv2d)\n",
    "print(conv2d.numpy())\n",
    "print(conv2d.numpy().reshape(2,2))\n",
    "plt.imshow(conv2d.numpy().reshape(2,2), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 합성곱 연산 ( image 개수 : 1, 채널수: 1, 필터수: 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant([[[[1],[2],[3]],\n",
    "                      [[4],[5],[6]],\n",
    "                      [[7],[8],[9]]]], dtype=np.float32)\n",
    "print(image.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = np.array([[[[1.,10.,-1.]],[[1.,10.,-1.]]],[[[1.,10.,-1.]],[[1.,10.,-1.]]]])\n",
    "print(weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_init = tf.constant_initializer(weight)\n",
    "conv2d = tf.keras.layers.Conv2D(filters=3, kernel_size=2, padding='valid', kernel_initializer=weight_init)(image)\n",
    "# print(\"conv2d.shape\", conv2d.shape)  # (1,2,2,3)\n",
    "# print(conv2d)\n",
    "\n",
    "feature_maps = np.swapaxes(conv2d, 0, 3)\n",
    "# print(feature_maps.shape)\n",
    "# print(feature_maps)\n",
    "for i, feature_map in enumerate(feature_maps):\n",
    "    print(feature_map.reshape(2,2))\n",
    "    plt.subplot(1,3,i+1)\n",
    "    plt.imshow(feature_map.reshape(2,2), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "weight_init = tf.constant_initializer(weight)\n",
    "conv2d = tf.keras.layers.Conv2D(filters=3, kernel_size=2, padding='same', kernel_initializer=weight_init)(image)\n",
    "print(\"conv2d.shape\", conv2d.shape)\n",
    "print(conv2d)\n",
    "\n",
    "feature_maps = np.swapaxes(conv2d, 0, 3)\n",
    "for i, feature_map in enumerate(feature_maps):\n",
    "    print(feature_map.reshape(3,3))\n",
    "    plt.subplot(1,3,i+1), plt.imshow(feature_map.reshape(3,3), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 합성곱 연산 ( image 개수 : 1, 채널수: 3, 필터수: 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant(  [[ \n",
    "                         [[1,0,1],[1,1,1],[1,1,1],[0,0,1],[0,1,0]], \n",
    "                         [[0,0,1],[1,1,1],[1,1,1],[1,1,1],[0,0,0]], \n",
    "                         [[0,0,0],[0,0,0],[1,1,0],[1,1,1],[1,0,1]], \n",
    "                         [[0,0,0],[0,0,1],[1,1,1],[1,1,1],[0,1,0]], \n",
    "                         [[0,1,0],[1,1,1],[1,1,1],[0,0,0],[0,0,0]] \n",
    "                      ]],     dtype=np.float32)\n",
    "print(image.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maps = np.swapaxes(image, 0, 3)\n",
    "print(maps.shape)\n",
    "for i, map in enumerate(maps):\n",
    "    print(map.reshape(5,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = np.array( [ \n",
    "                     [[[1],[0],[-1]], [[0],[-1],[0]], [[1],[0],[0]]],\n",
    "                     [[[0],[-1],[0]], [[1],[1],[1]], [[0],[-1],[0]]],\n",
    "                     [[[1],[1],[0]], [[0],[-1],[0]], [[1],[0],[-1]]]\n",
    "                   ] )\n",
    "\n",
    "print(weight.shape)\n",
    "# maps = np.swapaxes(weight, 1, 2)\n",
    "# maps = np.swapaxes(maps, 0, 1)\n",
    "\n",
    "maps = np.transpose(weight,(2,0,1,3))\n",
    "\n",
    "for i, map in enumerate(maps):\n",
    "    print(map.reshape(3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_init = tf.constant_initializer(weight)\n",
    "conv2d = tf.keras.layers.Conv2D(filters=1, kernel_size=3, padding='valid', kernel_initializer=weight_init)(image)\n",
    "print(\"conv2d.shape\", conv2d.shape)\n",
    "feature_maps = np.swapaxes(conv2d, 0, 3)\n",
    "for i, feature_map in enumerate(feature_maps):\n",
    "    print(feature_map.reshape(3,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 합성곱 연산 ( image 개수 : 1, 채널수: 3, 필터수: 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = np.array( [ \n",
    "                     [[[1,1],[0,1],[-1,-1]], [[0,0],[-1,0],[0,0]], [[1,1],[0,1],[0,0]]],\n",
    "                     [[[0,0],[-1,0],[0,0]], [[1,1],[1,1],[1,1]], [[0,0],[-1,0],[0,0]]],\n",
    "                     [[[1,1],[1,1],[0,0]], [[0,0],[-1,0],[0,0]], [[1,1],[0,1],[-1,-1]]]\n",
    "                   ] )\n",
    "\n",
    "print(weight.shape)  # (3,3,3,2)\n",
    "maps = np.swapaxes(weight, 1, 2)\n",
    "maps = np.swapaxes(maps, 0, 1)\n",
    "\n",
    "for map in maps:\n",
    "    map = np.swapaxes(map, 1, 2)\n",
    "    map = np.swapaxes(map, 0, 1)\n",
    "    for filter in map:\n",
    "        print(filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (3,3,3,2) => (3,2,3,3)\n",
    "weight = np.array( [ \n",
    "                     [[[1,1],[0,1],[-1,-1]], [[0,0],[-1,0],[0,0]], [[1,1],[0,1],[0,0]]],\n",
    "                     [[[0,0],[-1,0],[0,0]], [[1,1],[1,1],[1,1]], [[0,0],[-1,0],[0,0]]],\n",
    "                     [[[1,1],[1,1],[0,0]], [[0,0],[-1,0],[0,0]], [[1,1],[0,1],[-1,-1]]]\n",
    "                   ] )\n",
    "\n",
    "maps = np.transpose(weight, (2,3,0,1) )\n",
    "\n",
    "for map in maps:\n",
    "    for filter in map:\n",
    "        print(filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "weight_init = tf.constant_initializer(weight)\n",
    "conv2d = tf.keras.layers.Conv2D(filters=2, kernel_size=3, padding='valid', kernel_initializer=weight_init)(image)\n",
    "print(\"conv2d.shape\", conv2d.shape)  # (1,3,3,2)\n",
    "feature_maps = np.swapaxes(conv2d, 0, 3)\n",
    "for feature_map in feature_maps:\n",
    "    print(feature_map.reshape(3,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 고차원 텐서 연습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(12).reshape(3,2,2)\n",
    "b = np.swapaxes(a, 0, 1)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(12).reshape(3,2,2)\n",
    "b = np.swapaxes(a, 1, 2)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(12).reshape(3,2,2)\n",
    "b = np.swapaxes(a, 0, 2)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "a = np.arange(24).reshape(2,3,2,2)\n",
    "b = np.swapaxes(a, 0, 2) \n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(24).reshape  (2,3,2,2)\n",
    "b = np.swapaxes(a, 1, 3) \n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant([ [ [ [4], [3] ], [[2],[1]] ] ], dtype=np.float32)\n",
    "pool = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='valid')(image)\n",
    "print(pool.shape)\n",
    "print(pool.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 맥스 풀링 테스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant([[[[4],[3]],[[2],[1]]]], dtype=np.float32)\n",
    "pool = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='valid')(image)\n",
    "print(pool.shape)\n",
    "print(pool.numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant([[[[4],[3]],[[2],[1]]]], dtype=np.float32)\n",
    "pool = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='same')(image)\n",
    "print(pool.shape)\n",
    "print(pool.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = tf.constant([[[[0],[1],[2],[3]],\n",
    "                      [[4],[5],[6],[7]],\n",
    "                      [[8],[9],[10],[11]],\n",
    "                      [[12],[13],[14],[15]]]], dtype=np.float32)\n",
    "pool = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=2, padding='valid')(image)\n",
    "print(pool.shape)\n",
    "print(pool.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = tf.keras.datasets.mnist\n",
    "class_names = ['0','1','2','3','4','5','6','7','8','9']\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.astype(np.float32) / 255.\n",
    "test_images = test_images.astype(np.float32) / 255.\n",
    "\n",
    "print(train_images.shape)\n",
    "img = train_images[0]\n",
    "plt.imshow( img, cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv2d.shape (1, 26, 26, 5)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAABcCAYAAAB6Hp6BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6CklEQVR4nO19a4xk51nmc+p+v3dVX6p7bu7xeDIT2yFKAoEoyirgRECw0EYkCqylIFvIKHHYmFgRoEj8AH5sxPJnlawAZZNIBGIQDkSBCDZIKwKatfHGmYwznpt7+lbVda9zTp26nv3R87z9VXWPPdOX6q72eaRSzVRXd9V5z/e93/s+702zbRsOHDhw4OD4wHXYX8CBAwcOHOwvHMXuwIEDB8cMjmJ34MCBg2MGR7E7cODAwTGDo9gdOHDg4JjBUewOHDhwcMywJ8Wuadpjmqb9WNO0a5qmPbdfX2qS4chkZzhy2Q5HJtvhyGR/oO02j13TNDeAqwA+CGAZwCUAH7Nt+0f79/UmC45MdoYjl+1wZLIdjkz2D3ux2N8F4Jpt2zds2+4A+AsAH9mfrzWxcGSyMxy5bIcjk+1wZLJP8Ozhd+cA3Fb+vwzg3W/0C+Fw2E4mk3v4yKONVCoFy7KgadqGbdtTuAeZJJNJe25ubjxf8JCQz+exvLxsKS+9oVxSqZSdz+cP/osdIhYWFrC0tHTPMgGARCJhz87OHuwXO0TMzc2hWCyqL73ldQqxsrJSuqNT7gl7UezaDq9t43U0TXsSwJMAkEgk8KlPfWoPH3m08YMf/ABXr17FpUuXXldefkOZzM7O4pvf/Oa4vuKh4Dvf+Q4+85nP6CMvD8lFlcnc3By+/e1vj+vrHQr+7u/+Dr/xG7/xhjIBhuUyPT2Nr33ta+P4eoeC7373u/iDP/iD0ZffVKc8/fTTY/h2h4vPf/7zr7/5u7awFypmGcC88v88gNXRN9m2/WXbtt9p2/Y7w+HwHj7u6CMej6NWq6kvvalM3grWRi6XAwCf8tI2uagySaVS4/x6h4KZmRngTWQCvLXWSi6XQ6/XU196y+uU3WIviv0SgEVN005pmuYD8CsAXtifrzWZyOfzKJfLAOBzZLKFixcvAkDAWStbePjhhwFHJkM4f/48Op0OHJnsHbtW7LZt9wD8JoB/AHAFwF/atn15v77YJMLtduMjH/kIAJyFIxOBx+MBgCU4a0XgyGQ7PB4PpqenAUcme8ZeOHbYtv1tAMebDL1PnDt3DgB+aNv2Ow/7uxwx1B2ZbIMjkxFEIhHYtn32sL/HpGNPiv2oQ9M0edi2Ddu2oWmbMV+/349QKASXywW32y3vA4BGowHTNBEKhRCLxeD1ehEOh+FybTo4g8EAuq7Dsizoug7TNA/tGu8FrFXg9fG1Xq+HwWAAt9sNj8eDwWCAfr8Pt9uNQCAAYFMW3W4XHo8HHo8HmqbB5XKJ3FT0ej35ffWzjhooj9FnAOj3+7BtG91uF91uF4PBYIj31TQNXq8Xmqah2+2i3++j0+nAsiz5ucvlQigUgsfjQSgUgtfrhdvt3iavowbuERWDwQC2baPVaqHb7cLtdsPlcsGyLDSbTXg8HsRiMdi2jWKxCMuyEA6HEQgE0Ol00Gq1AGztxUAgALfbDb/fL/IJhUKHcblvCspiMBjImgcg+qTdbqPf7yMQCCAQCMie8Pl8iMViAIBCoYBWqyX7IRwOIxaLwefzIRwOQ9M02TfVahWGYaDX66Hb7e7pux8rxa4qMD6ojLhh+Xo0GkU+n4fH45Gb4vV6AQCvvfYaVlZWkMlkcObMGcRiMXkvsLn5b9y4gVKphJWVlSOt2NXNysXF11qtFjqdDsLhMNxuN/r9Ptrtthx6g8EAq6uraDQaCIfDCAaD8Hg8otgoW6Lb7YoSpKyOGlRlPhgMRHHx9U6ng36/j2azCV3X0W635f5qmga3241wOAyv14tGo4FWq4V6vY5yuSzry+fzYWZmBuFwGDMzM4jH4/D7/UdasVMe/X5/aJ1Q6RQKBei6Dr/fD7/fj42NDdy8eROhUAhnzpzBYDDA97//fRSLRczPzyObzaJSqWBjYwMAxHhIp9Pw+/1IJpMin6Oo2FWlTtnQYHG5XLJ/LMtCIBBAOByGz+dDIBBANBrF2bNnMRgM8K//+q8wTVMOhkwmgwceeACxWAwLCwtwuVwwDAPtdhuvvPIKVlZWYBjGW1Ox8+QEtjYbha5a37Q83W63WF98hEIh+T0+eMOArUXdarXg8XjQbDbF8hoMBqLE+v3+2K9ftapUb2Qn9Pt99Pv9IYuDf4M/syxL5NLr9eByudDtdndUeF6vFz6fTza4arnzYDwMa129b+qmBLas8H6/L8qLD3VdqF5MvV5Ho9FAp9NBu92Wv+t2uzE1NQW/349KpYJms4larYZSqSSHXSAQQDwel8PyML2X0bXBe6xeM+ViWRba7bbsn8FggFarhV6vh1qtBsuy4PV64fF4UKvVoOs6ut0u1tfXMRgMYJqmWJudTke8HipDfj6VJA2Ew5AJrXD1EAO2jEKv1wuXyyXf1+v1igFIPdHv99FqtcRDsW0blmXB4/HAMAz0+31UKhWUSiXE43GEQiG0Wi0UCgUYhoHBYABN02AYBjqdDgqFAqrV6r7olCOv2EcVFl0d3gCXy4VoNIpgMIhAICD0isfjgc/nQzabhcfjgWVZ6Ha7MAwDpmkO3UgeBqpr3ev1UKlU0O/34fP5sLq6Cq/Xi2g0CrfbjWKxiGazeSjWuup9eDweUVCqrLgALcuCZVlwuVwIBoMAtha2ZVnodDpoNpvodrvwer3w+/3yb8rY5/OhWq2KxRYMBhGNRuVwJO0QCATQ7XbRbreFmhgXeO39fl8UFw8jXdfR6/Wg67rQJnzU6/Uh+VEBlUolVCoVUYw8zL1eL86fP494PI5bt25hfX0d9XodpVIJXq8XkUgEkUgEwWAQ/X4f+XxeDILDABUtlZhlWahUKuh2u7AsC71eD6Zpot1uY2NjA4VCQfZVr9fDxsYGer0eAoEAPB4Per0eer2eeG62bePq1asAIHQdFV6v15P1E4vF5HdI36TTaUQikUORCY0Ur9crhz0A+Hw+uN1uJBIJhEIhWU+hUAhTU1ND112r1cTwc7vdKJfLWF5ehs/nk7116dIlrK6u4uLFizh16hRu3bqFl156CQCESeCByIM2m80il8vt6dCbCMVOJUbrkCcnsCkcWo9+vx8+32ZqMF0/9ZkWF0/iTqcjz5qmwTRNdLtdoShoDfNg8Hq96HQ6cLvdaDab8r5xyoLPvV5PvAdaD1RAvH5N09DpdOQ7c3PzffwZLTWfzyc/8/v90DQN7XZbuNJmsyn/t21brI5AICB/V41VjFMmtKxoYfd6PXmmQm82m0NKvdVqyeFNxU45lUollMvlIde72+3C7/cP/X3+LVr19OR2MhzGCe4b3mPeF8MwUK/X5b73ej2hAqrVqtRh0GurVqvo9XpCNdBACoVCiMfj4tVqmoZ0Oo1gMCiGF40wn88nNB5jDsFgUAyCccqEHgvjBeq+ADYPJxpF9Fr4ft5XPrfbbXmdhyZ590qlAsuyUKvVUKvVYBgGLMtCo9FAqVQaMhhM00S/35e4w15pGOAIK3ZeOAXHxRCPx3H27Fl4PB602205bXkDuBDb7TYA4ObNm0PulvpeuslcxK+99hoajYacnrROVL4+kUiI1UFenofJQcuD34ueQjAYRCQSQa/Xg2VZQquQTgEgbjGvXY0lNJtNdDodmKYJy7JkI9K6t21bDrtyuYxmsykHSCQSwcrKCsLhME6ePCkc/DhkocqEBxuD2RsbG1hZWRnyRHidPJi4BgzDwMbGxpDrS+quVquhXq+LUhqVTSwWQyKRgGmaCAQCYpGSY81ms2KR+v1+OXDHJRdazGtrayiXy6Ko6vU6rl69Kt4MPRHuN9Wbs217iP+mN8vEgmQyKbSK3+/He9/7XkxPT6NYLKJWqyESiSCTycDv9yOVSsHv9yOdTiMQCCCTySASiYihMA6ZqEq32Wwin89L7IxGDfVCq9VCo9FAuVwWms3r9aLb7aJUKgmFB0D0QTQaRTqdhmEYuHbtGgzDwNLSEprNJq5evSpJGY1GA4FAAIlEQvSKx+NBMplEKBRCNBrd8/UeecVOa8rj8Yj1HI/H4fV6ZfHScqa11ul0oOv6UEYDA32jSh6AnLarq6sol8vw+/3wer1ot9totVqy2MmdMaKtBk3GJRMGOFutFlwuF/x+v8iIslCzONRYQLvdhsvlkoyXnRQ7uUXSVZStyqkyoEhqKhaLiRVDq3fclhgPt0ajgfX1dbTbbdTrdXS7XdTrdfR6PVHs/B1d11EqlYTrZNYGYyrNZlOUss/nkzVEyoq0lMfjkeAoA8+hUAjBYFBoGLrd45IH14RhGKjVaqLYK5UK1tfXxVsFtgKEfr8fgUBAlD2wFTeh8lf3VCAQEEszGAxiZmYG8/Pz4uHRavf7/YjH46LQaaAFg0GhQO8WI9ov0ADkHlHpRwY9yZHT09V1HfV6HRsbGyIjy7KwvLwsXLrL5RKvZHZ2Ful0WigsGhvdbhe6rqNSqQxlUAEYysqjHPcj9nAkFbsaoa/X63ITOp0OstnsUNoUAHGN+R4uTCoz8li0UGzbltOVFhlPXVrwPEmptPmaz+cb2tjjVGCkG6rVKtbX1yXNimA2B5Uc+VNeOzcQrQ2msBGJRAJzc3Pigo4GxKjUvF6vKC4GUHnfDnqDqlDvFQ+TRqOBGzduoNVqiffGFDLyntxMTFcFIIqZsqE3NxgM0G63EQqFcP78eSSTSeTzeSSTSdi2jXg8Lt9FDdjn83lEo1FRGOOUC70Kn8+HRCIhgWLKgMqbsQceOFTIPBjUuAD3ZDgcRiQSQSKRkJ8Hg0HxajRNQyQSQSqVkoAj04V9Pt+QTKj0xiUb3tdyuYzbt29LTIX7hAed6tmTFfB4PEPX2el0JOgZDocRCoWQz+dx4cIFCab7/X5kMhm4XC7EYjHxfmzbFllQ7rZtIxAICI21VxxJxQ5s3QTDMMR1qlarqNfrSCQSssAoZDXTAdhaiKQSKpUKWq2W3LDp6Wmk02kJ/lExqEFZNXJPLo68HBXcuINiqqXJQA6/z2AwQKPRQLvdllS9RqOBer0+ZLFQ+fNADAQCYvlns1kAW1kwVOzkz3n9akyD/Dq/37iVu/owTRNra2vQdR21Wk08DB7yauCP8RTSU9zk6joib+/xeLCwsICpqSlMTU0hEonA5XIhEonI3+P1e71eTE1NiXLlITlOmdAgiUQikp3CVFb1Wqm41CwRGjWkWdTsEFJurOugpUlDi0YXExqYSsvPZaooPexxZ5X1+300Gg0UCgXZN1wzvV5PYkuMBTADjHoC2PRUer0eisUiDMOQepdsNovTp0+jVCrh8uXL8Hg8yGazYgDx73DPkv6h4aV+x5GeOfeNI6vYmdkSCAQQiUSEhzNNEzdu3JDFo2a8RCIRxGIxsSyBrXTAUqmEarUqC4qWWjwel00Qi8XQ6XSES1UtNy5a/kzdIOOCGhBrtVowTVOKRJiFQXePYHCPsqTlrmaB0HoHNq12HnC0qtRYhHpfyJvGYjFR8qO57QcNKhdy2fF4HGwMxetmpobKIzOgSEptdnZWMqt8Ph8ajYZQVYZhIBaLYXZ2FtlsFrFYTDKMqNDUA57UDGUxTqUOQA5hWoahUAj1el3W/NzcHEzTFBlQ8TCgrKZr8pHJZBAKhSTOQ46dsiQdoVqjfE1NR+a6Y+bZuGSjxpYSiQRyuRwSiYTQQfF4XAyhfr8v3jrXdzgcRjabRafTkdoVXdcl59wwDCSTSayurkpqaK/Xk3gMEzhUOlilt0ZjHHvFkVTstBhs20Y4HJZTTdM0NJtN/PCHPxzi2pn1cPLkSZw5c0asaTV/tlAo4Pbt21IpyoKL6elpTE9PIxgMIplMwu/3D30XNSMHwJByOwxrne5hs9nEysoKrl+/LoGbXq+H9fV1dLtdJJNJlmeLHBnsUivbeEhQsWcyGTk0XS6XUFvkE2lxMMNBDSryPeMELUq6yJlMBrFYDG63Wyx1KhFa4zzcqcDi8TgeeughxONxpFIpBAIBFItFSXlcWVlBOp3GyZMnkcvlhIdnEZeay8/1QsvvsMD7zlTDRqOBWq0Gr9eLxcVFmKYp8YW5uTkkEglsbGygWCwikUjgxIkTknXl9Xrx8MMPI5fLoVwuo1KpiMIitacG3FXKglQlD2DVUqdCGwdo7Hk8HkxNTQ3RSkxfNAwD6+vr0HVdPLJgMIhUKoVEIoF8Pg9d13Hz5k00m02hiSnrYDCImzdvwjRNGIYhSQbpdHpIedP7URW7+tgPHEnFroI8XTgcRjQaFfcZ2OTWqeQYEDFNE5FIRHg8pgXG43GYpinBrXA4LO55tVoVy/RunLnq7vP/44aa7hmNRhEOh6UsebT4iAEayk7l41k4Q9pGLe5iWictTlpho1SUWk8waomME9wsPFgikQhyuRwMw4DL5RoqTuL7mb7IoBmtsWQyKYqdVrfP50Or1RL6j8FStYhHLf6iYj9sUGHSQGIsic+apiGbzcrhxsweprq2220JjpOqUz1oWuIulwvJZHIo8K6mevJZper4PE7KDoB4p2rwljEAdW0DkOtRKRS1riMYDGJ+fl4MBtK7lB2hZqWp1jjZgFGLfb9wpBU7e26o/SV0Xcfq6qq4TZ1OB6VSCbquIxgMCpXAnt7ceGfPnsXc3Jy4T8zs0HUdr776KoLBIKanp++5vHncm5dejMvlwtTUlFhJpFVYhOTz+SSIWqvVcOLECWSzWXGdWUQDAMViUbjoer2OQCCASqUCADhz5ozwoztZ4WruL7/fYYCbgtk+c3NzeOc73wnDMFAsFsWSGgwG+Ld/+zfcvHkTuq4LZ5rJZDA9PS0WaSKREDk0m00sLy8jEAhgZmYG2WwWiURCNuob4agod3qW5IBZHDQYDPCOd7xD6Jd2u414PI6TJ0+iWq0OtQvgoeZyuRCPxyXFttPpIB6P48KFC/B4PFhdXYVhGDt+Dz6PKvhxgkZIJpMRqomGHFNcf/SjH6HZbAqFFY1GkUgkJAYFANFoFH6/Hw8++CDS6bQYmCyQ5Od0u12Ypol6vb7tu6jXfxCyONKKHdiiPvx+v9AyrC5lgEFVPrSYyJuzeIYCJ9dGq9/tdg9RE0dhQ94NvEbKgrnUnU5H6CeVR1eLleiSU7FrmiaBU1q7bE5EC0Y9TI4qeHADW9kG8XgcHo9HFDApKXLjzNFWqSUWzIRCIeFVXS4XdF1HMpkU2anBxEmBWixERa82KqOcVAubSk8N0KueIa9fpTjuZe8chkJXwWtQ22FQJ6gZLvRu1BREyoQenUo5qY3eGLfgoXEYa+VIK3ZuPgbHyINHo1F0Oh2Uy2W0Wi2EQiHUajWkUilZiIy+h0IhqTJrNBpIp9MAIAEPFmMcBmd+v6AsGLSMRqNIpVJCT+m6Dq/XK7nKjUZDFHgsFsMjjzwifCIALC4uStEKc7ZDoRB8Pp/k+97NYj9KUNPS/H4/FhYW0Gq1EI1GoWkapqY2R0UuLS1JFaCu60IncJPSO2RgPJlMysZl9sth9AbaLUiBkCdneh0tZzXFk3SmYRjw+/2Ym5sbqtEgNcWgIOVA79Dr9U6EbJi1xMNLjRd5PB6cO3cO09PTyGQyiEajmJ2dxYkTJ+T3WW3KWEGhUBg6INiq4KGHHhKq5TBwpBU7QcuRpyKryMgPMx9XXVy0Qr1er/R6IH3BFEmmvKmfc9SheiMMtlAJky9lKTgXGxdzMpmEz+eTrBlaIqxgHQ2ATsJGJdTMHXp0LMhKJBIANlP/+DM1pZFeofqgHJiTzWDofmUtjAtqDILxEspKtbKZ+sjeJ4wnANvXwaiyYhBwktaLmuWlpvCy3S4tbj4I5puTBmX6LH9GHj4ajUp1815TF3eDI6/Y1YXIRUiqgEEfFhfouo5yuYxut4t8Po9UKoVz585JAMztdgu/rgYEqciOaqvZUahuITMzmHOuljyrqZu0uEKhEFKplKSeDQYDKZ5QC37Uwp9JgNpECdiUEbNjeGjNz8/DMAxUq1W89tprAABd1yXXPxwOS7BRrVfIZrMih0lSXsCWl6cW9JG+YlXt6urqkNWqFiDpuo5Wq4VgMIherydV16onoOv6WCtr9wp+d7XpHy1xUlalUgmWZUljr2QyiVOnTqHb7UrSAQ0jtvIYrWw/zMPuyGsyNdhCi4nWFN1kZnywnDwcDku3PWbTxGIxaTfAjoxc6JOi0Al+X0btgc0NzJL6UCiE69evC6VARc0sFyo8LlC1qdqkQk0V47Wo68PlciGVSiGXy0leMQBp5MWUz3a7LcodgMQe1M+ZJKgFdyrnywOPwWW1NoM8s23baDQaAADDMIYyRVSMu5PnXqEGcXk/mRJLT4aFkWzJMDc3h5mZGan5YLrsaIzuqGCiNJrqAjJX2Ov1Ip1OS0B0Y2MDjUYDr7zyCsrlMh588EH0ej0kEglxq9migIq+VqtNjLWhYtSidrvdksaZzWYl/fPGjRvw+Xy4deuW0DFqbxNuflovqhcziVADe+r/k8mk1DosLi6i1WqhVqvB4/Hg1q1b0mKA8RwWK1Hxjbv4ar9Bw4j3ln344/E4FhYWpHe42sGTtQArKysoFovIZrPIZDIAtop+aChMItTUTJfLJemdjLsxZZpZVgBEiTNrhokL7JxJWuswA8UTpdhVkEsFNt3jYDCIarUKl8uFRqOBl19+GaVSCe95z3vg8XgwNzeHWCwmk3GYhsT/T6JiB4bjArTCACCXy8GyLLz66qu4du0aPB4Prl+/jlwuh1wuN9TSmFBzag8rL30/wQOK93ZqagrRaBSrq6tYXl7G7du3cevWLfR6PSk60XUd0WgU8Xgc0Wh0KPbAjIdJhkoVtFotDAYDSfHs9XpYXV2VOAwTC9irnT1VWLFKQ4JpsZMMHnj0Yvr9PkKhECqVCmq1mrQdULu6sugvkUggkUigVCrh5s2bQuscpk6ZWMWuulPM9IjFYsjlctIAqlar4cqVK9IrJZfLyftZZOH3+yUjgJ0Rx1kRt9/w+/2wbVtaypZKJaRSKbhcLiwtLcGyLOTzeViWJRwhM0Fo0Y0q+EkHYwnMfpmamsKpU6fQ6/Wk9L1Wq8m1swSfHh5pCrVXDq3eSZWPmrrI/RMOh6V/Eq3ScDgMv98P0zSHPBzSet1uV1pKkJoYrXGYFKg6hfuIXUyBLcpJrbTlpCSmS9LK55AOVtmOW59MrGIHtlzsQCCAYDAo02pWVlawvLyMVquFb33rW0gmkzBNE2fPnpV+H8wE0XUdLpcLzWZTCizIt00a2FogHA7j9OnT0tmPAeNLly4hlUohEolgamoKs7Oz0jucQdhgMCgBouNgvdPqVPvJLC4uSvXha6+9BsuysLa2Jm0nVJ45n89L+TkAKVw56vn99wK1RsTtdiObzUrufqFQgMfjwfT0NABgZWUFtVpNrHdWP7MmghOGmP89aXErgjqFNQ3siGnbNqrVqswmYBOwcDiMixcvSl0Nq5XV3k1qo7VxYTKlfwc8BWkh+Hw+GU0WiURkLNpgMJCRVXw/i3FoqWiaJnnf/J3DKHveK+gSs/Q7Ho8jmUyi2WyKgi8UCjJViS0EuBmZcaQ2sKIcJlm5M+2T+duJRALpdBozMzNSjcxMCZaJc6AEM63IvVNpqcp9EmWjcu60wBmjobJXA9Gk+bh21N783EcE9+SkyWVUp9CjZVqxOoSnXq9L5szGxsZQuiyTFth1dXSG8EFjohU7wUEJLK7xeDwi+CtXrqBQKOCFF15AJBLBhQsXcO7cOSwsLODixYsIhUKYm5sDsHlKs7kWOcVxjr7bD5AuYBMw8qFU5u12G9/73vfgdruxuLiIdDqN06dPY2FhQdqOut2bMx/ZBnh0UU7aZgUwlM3AsW5M56tUKlJKXi6XYZqmzDLd2NhAp9MRaisajWJ+fl7a0qrB50mEmtrKwCE9EnXo9cmTJzE7O4vl5WWsrq4OlcxzHrBlWQiFQtKagUHFSQTXOtsHqD2parUaer0ebt++LRlVxWIR0WhUWjrPz89L6qjf75fReOOy3I+FYifUTn9sKcoOdeVyGfV6XRQe3cxEIiFDahmMZUYEBzNMGmgpMBiUTCZlmvpgMJBxcJxZyTJ6j8eD2dnZoUZR7MvNbnyTqsCArb4yjK1Eo1HMzMzIsHIA0ilUnbTDodacIM8sLD4mnZLhPaWXxvWv0hAsWCqXy0Ol8wzCqmnE6r6ZVMVOMF0WgASU/X6/ULWdTgf1eh3FYlFSIFkjonq+LJYDxpMyeywUu1pdR5qFww7In9++fRuNRgPXr1/H0tISstksLl++jHw+j8ceewypVArJZFL6s+dyOWmLO2l0DLB1yCWTSVHwAFCv16WoZH19HUtLS1hdXcWrr76KkydPotlsIpFI4MyZM0O9pDnzlMp+EqFOCmLTsAcffBC6rksdxM2bN1Gr1XDjxg2xxq9fv45IJCJ92TudDtLpNPL5vMz05IaeRNDjUGkZBv9IVzKgns1m5XX23SkUCgAgrZ6Xl5exvr6OmZkZnDx5ciLlotKPzP5hkVY0GkW73UYul5N4xJUrV5BOp9FqtaTJWiQSQT6fl71HBT+O+N1k7tA3APPbGVDlMIpqtSpVh+xoyJ89+uijcLlcMkyARRtUZJOo2LlZaXnbto2ZmRkEAgHZiIVCQVoW0yplVlEul4PL5ZLJ88wsmvS0NgDSJM3tdiOdTiMcDg/NwaSSr1arME1TZETqZnp6GpqmIZFISMOx4wAqYHLELLdX4yzhcFhoLHXEnW1vDZ7npDKmDk6iYie4j5gJNBgMJBuIhUylUgnlchkAJLanVsCz3oYtTMahU47HirwDtY8HFyUHL9i2jfn5eaysrGBtbQ22baNQKMDr9eLatWvSypVUTSQSkZad5NEmUcGzG10ikcD8/DxSqZS0X4hGo6hUKigUCpIFwZQ327alqVomk4HP50MqlRKefpJ5Zaah0RpjK+R4PA5N2+pTnkgk8Prrr6NaraLVamFlZQXlchmhUAilUgnhcFgmzbNb4KSm+gFb8RlmBXFeAccDMmjIGbCBQEAKmdTGWOq80EajMUTtTRpUncJDijUNqVQKsVgMxWIR5XIZ09PTOHXqlHTM1HVdJpyVy2XUajWRL2V2UDh2ip3CZyYE+7JHIhFYliX9yjkZx+VyydSTUCgE0zRx6tQpseSi0ejQNJ5Jg9rIKBAIoN1uw+fzwTAMBAIBlEolNBoNbGxsSEyC9E06ncbCwoLMf6Qly5bHk2qpqjM+ST/wMPP5fDBNUxR7r9fDf/zHf8CyLGxsbEjPnUqlghMnTmBhYWFb87lJBQ9rdjxUh5QwI0QdiE1LlD9nlgzBgCM5+0lV7CoVxWZqLJCkIZDNZjE9PY2FhQV0u12sr69LVh47rpbLZcTjcaTTaZHZQRmLk7kz3wRqfxmC+aXz8/NSkJJKpSTqzU5sXq8XpVIJmqZJr4jDLg/eLzADYnSCzsLCAsrlssyPZTM1ANjY2IDf75epVHRFJzEVdBS0xGiZ2vbmeDRWZKpDn8mLDgYDoegYIJx0S30UKq3CNr3sK8Th5uVyWTwZtVeMOkmJVamctDXJULNZer2eXB9HdObzeczNzWF+fh6maUqPHbaGbrfbqFQqQ11ID7KS+dgqdvUmaNrmsFq/349MJoO3v/3tqNVqWF9fHxrUUKvVpPq0Wq3KzMvjoMQILkS2sGX7YwAol8t4/fXX0e12ZT7szZs30ev1ZOoM2/yqynBSYdu2UAzsjFmv1yXARXpF7Z1C/tQ0TZimuS1v+TiA3Qq73S50XUe73UapVJJeKO12G8ViETdu3JC2yDQW2NaCXUSTyeSxkEu/3xdPVe2O6ff7MT09jWg0irm5ObztbW9DpVJBsVhEt9sVb6XRaEgCB2dITE1NHVjM6tgo9lEFQ7dSnc9JF4pWGNP9GBhjRgAb/9BdAibPxaY8SBGoKYvsRc7Dj5yqaZpSgDPqgpNDBSABtUmD2tFP5YtZ89DpdFCpVORw7/V60utfrcBVOyZOesM0YCsNVB32zQrsZrMpXDmHNHMgB/lzBknJsQPDFKD6GZMCdUauSt0BW22zORuC72ESQrPZFIucMuC/mWp70Gmgx0KxqzwYB1IzCs1CCY6zArbyvDkRhl3bWFnocrlkkTL/dNLcSSpuzkTl4mTfdlrczA6Jx+OyuVX+udPpwLIsmKYpQUI1R3eSwDViWZZcU6lUQqvVwvLyMgzDwNramkwEYkdMUjNqR0w+OAx5ktbGKJi332g0UKvVRLHTUm+32zKtjNRLoVDA2toawuEwUqmUHIIMurOVNmk7lXs/6lCNGuoBTmPz+XwyO5iHVavVQr1eh2EYWF5eFqWudgllpXMmk8HMzMyBr5fJ2pkj2MkCUC1v9THaAB/ANmuLSo6KjVTDJFkaqgWhBrVoodLN5jN/Tu/G5/PJIcC0LNXa2ilL4ChDvX88tCzLkvF4THstlUryzDGBbH6l9lRhIzE1eDhpKaCjnguVl9qTXrXUmeHBwiNmyqj7S7ViGZOhMTQJ+0eVCQDxalVPTW2xQLlR+TMmwbXCYsBWqyXTloCtVFIAB5rPPpGKffQmULkwQEoemK+TblGVG5Veu92GYRjodrsoFApSfEGLnpHvo6zAaGEAW9VxXGjciO12G/V6XXhTloOzHz03sNq9kN7LYDAQqwPYoqWOukz4YEYT+U329qjValheXpYWAuwKys3IYHIqlYLP55NipHQ6jUgkgtOnTyOXyx1pOYxC7XnSbDbR6XRELowb6LqOjY0NWJYlFrtpmmi323KYZTIZ5HI5ackRDAaRy+UQDofx8MMP4/Tp07K3jrpip3fLFE3uJdaBcJwih2osLS0NHX5U5oFAQKZLsS/70tKSxOzq9bpU9DoW+10w2rtEtTjpRnN4BHlDWqe0YNWxWFRwHCfHGaHsxz0JUA8u1ZrgLEsW4FCJk0el293v94dG46meDmU7CR7MqHfB3Gpa6ZVKBRsbG6hWq1hdXZVURsuyRNmp/Dmn1ieTSQQCARl0zJGCR3GCzt1Auo1NzqigTNMUekrXdWlwxbVC747WeSgUEnmQekkmk0LNZLNZ4ZsnIR6jerjcB9wDnP3KbCHKq1arwbIsqUqll0uakzEbejqUe7/fP3Av700Vu6Zp8wD+F4BpAAMAX7Zt+79rmpYC8A0AJwHcAvBR27arB/Eld1IkVORq8yKVWqHypqXOIBldTVoT6ntYWUl+/m7pSLVaDd/4xjck7e3d7343fvqnfxqmaeLrX/86AFzQNO27ByUTlW4BIJtUDZByY7JIgs2L6FargUMqJgZ3GJsg3QBAft/n80nvDBVra2t47rnnJFX0ox/9KH7t134NtVoNv/VbvzUWmQAY8sZ4r4vFIlqtFtbW1qSxV7FYRLPZRLFYRLvdlmEr7BLK9sehUAjpdBqhUAizs7Oi4IPBoOS+v9FBt7q6imeeeUa6/3384x/HJz/5SVSrVQBY1DTtNRzg/lED5pRJsVgcohCKxSIajYY8TNOU1Dxg81DP5XJDcqEHFwwGpQ321NQUQqEQwuGwKLKdZLO+vo7f+73fQ7lchsvlwuOPP46Pf/zjqNfrWFpawjhkot63brcLwzAAQPpIMZefgXXSUtwvLH4Etg6FVquFRqMh1r3L5cLc3NzQkBZa/weNe7HYewD+q23bL2maFgXw4p0N+gSAf7Jt+w81TXsOwHMAPrffX3CnnHS19Jl8ViQSEQEzg4PC5gnKEVfq0AC1UpU5p6PDFEbhcrnw8z//85ibm0O73caf/MmfYHFxES+++CIeeOABXLt27YcA/ukgZULemxYpA6IszqJCr9frQi+QiiH1NNppjvwfNy0LkwCI8uPghdEDz+1247d/+7fxtre9DYZh4Jd/+ZfxUz/1U/ibv/kb/ORP/iS+//3vH6hMVLnw0KJlxYDo7du3ZfwbUxbZ2IvBPY46i0QiSKfTSKVSmJ2dlb4fgUBAUmdZ0fxGcLvd+N3f/V1cvHgRuq7jwx/+MH7mZ34Gf/VXfwUATdu2Fw9y/6h7gjJZX1+Xg9y2bVQqFZkUVC6XxVJ3u92IRqPweDxIJBKIxWKIxWJS+8EOl9PT0+LJcHgL4xl3k8lnPvMZPPTQQzAMA5/4xCfwnve8B9/61rcQCoWg6/qByoTUJZU79w8LGllV6/F4pAUJjQAakl6vV4a06LouaaHValWSEQKBgHR7ZFym1WpB1/X9vqRteFPFbtv2GoC1O/9uapp2BcAcgI8AeP+dt30FwPewTzdB5UdJs4TDYRkqrKZn8T2qq0NOfSc+mQucU0+ArcIdtislV7+TZQpAFjiwyetns1nU63VcvnwZTz31FL7zne/sq0xUrpiBXaZs8nqorPms9sKhO03lzza8LCRRr5feD4cmpFIpBINBsdLulh3E6jtgc+rOmTNnUCgU8M///M/4yle+gi9+8YsHIhO6tqSTSCWo3PGtW7fQbDZRKBTQaDRQr9fRaDSGMqiYo0/rfGpqCul0GvF4XCzSbDYrCo10n5o2uhM4ihDYtAYfeOABrK+v4x//8R8BoHznbfsmFypypm9STvV6XZQ2B0aoFjv5YsZTIpGIyIVrPBKJiGJngzhSMVxDzChTK5RHMTU1hampKQCba+XUqVMoFov4l3/5F8Tjcb5tX2SiGkFc/+xmyjgauzICELpxY2NDssZ46KtDzSljfgaw2fabTfPYiJBxKnpL46Ls7otj1zTtJIBHAfw7gNwdpQ/bttc0TcvuxxdSN6xt28LzxuNxnDhxAgAkMGgYhriU5LG4WNlKVHWjLMsSDlmtkCPtwg3LKTB3rvkNvy9bEywsLEDXdbn5+yUT1bogJ87vR+6clphlWVhdXZUgGBUdx3epWT5MXfT5fNLMKhAIyJizdDqNQCAgz/eTe7uysoIrV67g4YcfRrlcFoW/nzJRc4c7nY40NKvVatJDXdd1tFot3LhxA81mUygppjtysEQgEMD09LRMTKIyJ7XA9DY2/GLFIb2De8Xt27dx+fJlPProoyiVSgDQ3W+58D7TS6ERwE6lAIRiKBaLElcwTVMygSKRCObm5uDz+aRFMfn0eDyOWCwmfXJIz7jdbjnkWOzF7/RGYGfRCxcuoFwuY35+nr+3bzqF94nN3PL5PBKJBKLRKFKplMSdVINwbW0NxWJR0jb9fj+mpqaGDk1W40YiEZl/SrlFo1FomiZxLeqkceGeFbumaREAzwN4xrbtxr1GdTVNexLAkwAkq0KFSrWwwY6madt6S7DlJbCV+aEWzqiTb3jKqs+qpUtlzrxkKnZaYfda6ttut/G1r30Nv/iLv3hX6/7NZDI7O7tNHmpxhPoaFwiDO7x2bk5y6iwgoSz4dyhb9SAj7cJAICdL0c3k4JJ7vd+GYeBTn/oUnnvuOWlXer8y4eATVSYqzUKF3u/3xdJcW1uTvjeVSkW8NNIKHHTAHHy626QPZmZmEA6HkcvlZMPH43GZHKTyrrvJaDAMA0899RS+8IUvIBqN7kouHFNHqLUKatCcB71aXMWYi5r6SZmwrTM7okajUZEPD3zGXGKxmLyXe0XNKrmfyWOmaeLZZ5/FZz/72V2vlVGdolIs3PM04EghkWZpt9tCS9H4UcdihsNhoSZdLpesOVKeVOistA2Hw0MxPvUejTuAfE+KXdM0LzaV+tdt2/7rOy8XNE2buXOyzgAo7vS7tm1/GcCXASCfz9sjPxuq7FKtZbrGtMJdLhfK5bIoOXWTc3NzMdFC5zNvBPOQSbcw44UBjfvp99Hv9/HVr34VjzzyCC5cuABg09Vmj4h7lcmFCxeGZKJGztmBkA2ZaG3S8iQYEGN2ByPyVGL0UFhgkU6nhSPmmLh4PI5QKCQFXXQh72fEWbfbxac//Wn8wi/8An72Z38WAJBOp6UA7F5l8va3v31IJmoqGg8wBkTZBuL69etYXV0VDpNeH6mGdrstB3Y8HpeCkdOnTyMSiWBhYUEyOqjMGU9gyivBOMa9otvt4sknn8Qv/dIv4UMf+hAAIJPJoNFoeO9HLufPnx+SS7vdlsO8Wq2Kl8IMr9FDjX1eeOgzxrSwsIBUKiV0J6tJQ6EQTp48KdY5KTk1S4S52qo87kWpd7tdPPvss/jQhz6ED3zgAwA21woV4m51ihpf63a7UvLPOAEA8T6XlpZw9epVicnY9lY/+mw2i/n5eQmEMjZh27YkGIxO0lIPBg5HV2s+xol7yYrRAPwpgCu2bX9R+dELAP4LgD+88/y39/vhaiENLUt1Ug9dRloFfNAC5w0kf86DQrVsCdUqV9MiWep7P5Fq27bxzW9+E9lsFu973/vk9fPnz+PFF1/kf3clk1FPBNjK9ODmJL3C61UDo1TolJE6UZ6HJukHWuzcxEzro5zuxzK1bRu/8zu/g9OnT+OJJ56Q1z/wgQ/gb/9WxLArmfDv01VmX33mnZM3pnJTD3kWjTDTiePKkskkksmkDPdOJBLSzZOWKxUAD7jdWF22bePZZ5/F4uIinnzySXn9gx/8IL70pS+l9yIX1VrnmiGPS2XOwJ5qTVNZkTZkpgtjR36/XzKCqLgY/ONeUeNg92OlUya///u/j1OnTuETn/iEvP6+972P8aldy4RUFPUED2FVyTKRgjRet9uVRmakG9W2G/w3AGET2BqAxqLat56/c5hpwfdisb8XwK8CeEXTtJfvvPZ5bCr0v9Q07ZMAlgD85/v9cPLjpmnKZB82zmEQg9YrCyja7bak7ZXLZWlDS2qCVioVE91FulWkYNSS+Pt1rW/duoWXXnoJ09PT+OM//mMAwGOPPYb3v//9ku4IoL4bmai0CocEUw4s62ZgipQLFRsDXzykXC4X4vG40A3kROkN0Tolh6hmA92vTF566SW88MILOHv2LB5//HEAwDPPPINf//Vfl3TH3cqESskwDKyvr6NSqeDll1+WvPROp4OVlRVUKpWhqmMqaAa0qLQZ6E0mk1hYWJDmcD6fTzKBRj243brSly5dwvPPP49z587h537u5wAAn/vc5/D000/jS1/6UuxOat+u9g8zlXq9ngRymfq6uroqdIxpmjLYHIAo68XFRWln3e12kUqlpF01vTquFVUOXJeqNXo/Suzll1/G3//93+OBBx7Axz72MQDA008/jSeeeALPP/88disT27bRbDaxsbEh35Xr2+12o1qtDhUUcRgIjQEAYtiwzS6vTc0Um56eFv0EYGivjqZSHhbuJSvm/wC42y7/T3v5cNU6ZeMl5mOzNwlPTSoxLmbLslCr1aTpv8/nkwWqTn9RLXS1Ac9eqklPnTqFP/qjP9rxZ08++SQ+97nP/dC27V3JRk3XpCwoJyp8NmJS06yYh6teO+knWuRqsIvPPETVDKHd4Cd+4idw5cqVHX/253/+53jooYd2LRN+J1pjam8OBup473mQqzn5zPBIJpOS8ZLNZhGPxyWjg++hEtgvvOtd78Lt27fv9uOrtm2/cy9/n7EpXrPaOkNNOWRlJI2ecDiM2dlZxONxaU8di8XEc0mn0zIij0pQrVrdi/J69NFHVc92CCdOnMCPfvSjxd1JY2uIinrdrVZL2nCrDf94v9U1T52hKnvSLzQU2IqYOuuge6vvBodaeapSJ1ROhUIBtm3jpZdeAjDcFW4nZax22qOLRLqBbjcDXwfdA3m/wAyY5eXloYIitbCIXCRHlHGx0l0mJ5pIJKQicGpqSvhTNXh81GXi9/sRDAbR7XaRzWbh8Xhw4sQJoeD6/T5mZ2dhWdZQgy6OcFPnv5JuicfjIhNa+JPWqTGTySCTyciaaLfbUhx05swZMZrUilE1G4zWKZtSxeNxJJNJ2T8AhOJUaYajpMBGQYNF7Uz64x//WLy6Vqsl2WBcV/TMaJWHw2Gha6LRqGT9cJ9YliU/v18aalw4Eoqd1oVpmkMpWOqi4k2gtaFaYlTsvEE8XfleWjJHWXmNgpQTi2hoxXMxDQYDuU4Gg9XUTQZEOeKPsypZWLHb7I7DAK+PnG+v15NZrLQkOZWGG5b51cwhBiA0QyAQkHhCLBbbNX9+mGBtB1saUNmQU+ZhT2qAvC+pqsFggFqthl6vh0gkIsH0WCwm1r/aKO4oKq+dQCuc/VjYidIwDKysrMAwDMn4ofXt9XoltsDYUygUwmAwQCaTQTabHQo6s+jxKMvjUBU7Aw/MwrAsC8lkUnJEGQC0bVtOTHKnbrdbLA717zBARkXHU3ZSlBi/Mwth2IiK8QDSBLQweIAxLZGFEZlMRibYMLsjHA6LHCcJPLjV/OCFhYWhVrA88NQOg5xhyjQ/Zn6oawXYPX9+mLBtW5SuGjw3TRMAJHBO5cN1pc7c5OBlFhq53W7JxOJhd9QVmAqW7Kte2mAwEO+FtB1jcmqpP7025u2ryRfMrCPFc9S9FuCQFTuVEV1mTrBhg38104VWPRcmgG2l7czsuJ8UvaMG8qXMIXa73cKj073kIgyFQtIelVYXqyTVbAY1dXESobbNpWVF3pcbk7JRhyFwAARzlFkuflygjq0rFotDWS/AVryFXiyNHu4fvo+prkynPar0wr2AXh2zvfx+vxSY8fBS9Ylaqa7OKmDMhp7zpMnj0KkY9fRTCwpIu/B1PlPx7cSXkyedVKUObLmSnO5Ezk9thaBOhSIVQ+8lkUgM0Vbq5p5UMHuK9Qj01rhegO0TeqjAmNFAr+84gZa12reGip2TsLgX1DiC+vvAcIrgYeVd7xfUQjYqa+oKWuqqPlFTqCkH1kxMmsei4lAVu1qRpd1pyK+mJKpQhXs3JTXJCp1QJ67MzMwAGFZazIIAMPQ8+pqqzCddJurINVbOKj1F7gpeN9fSpMthFCq1Mlq5udO9H71+HnTqTNdJVGIqRrt7ApsD2VV57HSN6v7a6XnScGT6sauZL+rzWw1qlo+DnXG/B/hxX0v0YnaL46DQR3Ecr+l+oI3z4jVN2wBgACiN7UPHjwyAsG3bU/fyZkcm23FHJq/f+b3jKpf7kgnwllgrjkx2xv3LZdynmqZp/3evRRlHGbu5Pkcm+/t7kwBHJtvhyGRn7Ob6HH/fgQMHDo4ZHMXuwIEDB8cMh6HYv3wInzlO7Ob6HJns7+9NAhyZbIcjk51x39c3do7dgQMHDhwcLBwqxoEDBw6OGcam2DVNe0zTtB9rmnZN25xAPtHQNG1e07T/rWnaFU3TLmua9uk7r39B07QVTdNevvP48Jv8nWMjF0cm2+HIZGc4ctmO/ZIJgOEOiwf1AOAGcB3AaQA+AP8PwPlxfPYBXtMMgHfc+XcUwFUA5wF8AcBn34pycWTiyMSRy+HKhI9xWezvAnDNtu0btm13APwFgI+M6bMPBLZtr9m2/dKdfzcBXAEw98a/tQ3HSi6OTLbDkcnOcOSyHfskEwDjo2LmAKhjZJaxyy98FKFp2kkAjwL49zsv/aamaT/QNO3PNE1LvsGvHlu5ODLZDkcmO8ORy3bsQSYAxqfYd2rWcSzScTRNiwB4HsAztm03APwPAGcAPAJgDcB/e6Nf3+G1iZeLI5PtcGSyMxy5bMceZQJgfIp9GcC88v88gNUxffaBQdM0LzZvwNdt2/5rALBtu2Dbdt+27QGA/4lNd/FuOHZycWSyHY5MdoYjl+3YB5kAGJ9ivwRgUdO0U5qm+QD8CoAXxvTZBwJts2XgnwK4Ytv2F5XXZ5S3PQ7gh2/wZ46VXByZbIcjk53hyGU79kkmAMbUtte27Z6mab8J4B+wGcn+M9u2L4/jsw8Q7wXwqwBe0TTt5TuvfR7AxzRNewSbLuEtAE/d7Q8cQ7k4MtkORyY7w5HLduxZJoRTeerAgQMHxwxO5akDBw4cHDM4it2BAwcOjhkcxe7AgQMHxwyOYnfgwIGDYwZHsTtw4MDBMYOj2B04cODgmMFR7A4cOHBwzOAodgcOHDg4Zvj/IzGBX/y2HcwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = train_images[0]\n",
    "img = img.reshape(-1,28,28,1)\n",
    "img = tf.convert_to_tensor(img)\n",
    "\n",
    "weight_init = tf.keras.initializers.RandomNormal(stddev=0.01)  # (3,3,1,5)\n",
    "conv2d = tf.keras.layers.Conv2D(filters=5, kernel_size=3, padding='valid', \n",
    "                             strides=(1,1), kernel_initializer=weight_init)(img)\n",
    "print(\"conv2d.shape\", conv2d.shape)   # (1,26,26,5)\n",
    "feature_maps = np.swapaxes(conv2d, 0, 3) # (5,26,26,1)\n",
    "for i, feature_map in enumerate(feature_maps):\n",
    "    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(26,26), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 13, 13, 5)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAABaCAYAAAChWQ3bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQJUlEQVR4nO2deWxVVdfGnw1SJrXQwWotAoqQIGJoGSQ4BYqiov2CFsEJkAimeTFiNCImGEMwr+Bf4osGIylGwytKjNiQfBY0IA5gIVirhLZiwUoFWjpZkSLs94/eXu8eai93OPfs3ueXGFjb1Z7N03MXh73OWktIKUEIIcQ9eiV6A4QQQiKDAZwQQhyFAZwQQhyFAZwQQhyFAZwQQhyFAZwQQhwlqgAuhJghhDgkhKgWQiyL1aZchprYoS4m1MSEmlwYF0X6hUKI3gD+A2A6gFoA3wohtkopf+zqa1JSUuSAAQMivaTvkVKiV69eOH/+/CkAVyAMTQYNGiSzs7O922QCCNQanAdwLcK4V9LS0mROTo6HO/SeC9UEAFJTU2VWVpZHO/QeKSWEEJBSXgPGFIXm5uZ6KWWmvh5xAAcwEUC1lPIwAAgh/gugAECXYg8YMAA333xzFJf0N42NjaisrMTJkyd/llK2h6NJdnY23n33Xe82mQDKy8uxYMGC38O9V3JyclBSUuLlFj1n3759mDVrVtiaAEBWVhbWrVvn1RY958cff8Tzzz+PtrY2xhSNkpKSI7b1aI5QrgTwS4hdG1hTEEIsEkKUCSHK2tvbo7ic/zl9+jT69esXutStJo2NjZ7tL1GcOHECAEJ/+IYuoZqcOnXKy+0lhN9++w3oRhNA1aW5udmr7SWE+vp6XHSR8kyZ9DGlO6IJ4MKyZtTlSynXSynHSynHp6SkRHE5Z/lHTQYPHpyIPXlKF+0apOYT1CQtLc2bjfmPf7xXUlNTE7EnzwjnPgn4JXtMCRLNEUotgCEhdg6AY9FtJ3ZcfPHFiv3XX38ZPg8//LBiX3LJJYbP1KlTFdt2k917770AgP79++PPP/8M/V++0iQc9PP4Y8fM7Q8cOLDb79PW1hb8feDcNvST5pwusebyyy8HqIlCZmam/jn1lSb6WXt6errh89RTTyn28uXLDZ8PP/xQsevq6gyfRYsWhbWnaJ7AvwVwrRBiuBAiBcAcAFuj+H7Ok5qa2hm4UqjJ34wePRoA+vFe+ZsbbrgBoCYKo0aNQnt7O6hJ+EQcwKWUfwH4F4D/B3AQwGYp5Q+x2piL9OrVC9dddx0AjAQ1CRI41zwK3itBqIlJ7969cdlllwHUJGyiOUKBlHIbgG0x2kuPIHBcUCGlHJ/ovfiMZmpiQE00Bg4cCCnlyETvwxVYiUkIIY4S1RO4F2iv5QEAnn32WcX++eefDZ9p06Ypti1R8Ouvvyq27fW1lStXKvaIESO63qxH2BKpLS0tit2nTx/D5+zZs4qt//kBQH8ty5bEHDlSfUAaOnSo4VNVVWWsxZNz584Za/rPs6KiwvA5cOCAYusaAR3vJ4fy4IMPGj67du1S7NWrVxs+iXjlLTSZ3Mn333+v2LoGgPnnaWhoMHwmTJig2C+++KLho2t+//33Gz4//fSTsRZPxo0bZ6xlZqo1Mk1NTYbPjTfeqNi2ezyc9/QvvfRSxR4+fLjhc9VVV3X7fQA+gRNCiLMwgBNCiKMwgBNCiKP4/gy8rKzMWFu/fr1i6+e/AIz+Itu3bzd8VqxYodi2/hu9eql/x+3du7frzXrE22+/bazphUu2fepl++Xl5YbPxo0bFfv8+fOGT6AMPEgiznb13Ej//v0Nn9mzZyu27cxy7ty5im07z1+wYIFi2wq+br31VsW2FY55gVaKjtOnTxs+v//+u2Lb8kOB99SDPP7444ZPoEVCECHM4uy8vDzFtuUqvOa9994z1g4fPqzYY8eONXz0OLN27VrDZ/PmzYr9+eefGz7z589XbF1HwJ67sMEncEIIcRQGcEIIcRQGcEIIcRQGcEIIcRTfJTH1RIheLAAABw8eVGxbIcJrr72m2HqSDwBKS0sV25U2pgUFBcbanj17FNtWiPDKK68oti3x17t3b8X267QTPXEa6KGhoCeln3zyScPniSeeUOwrrzTaTxv3ZDgJJlvy1wv05KmtXfHVV1+t2PX19YbPsGHDFHvQoEGGj54ktr0ooOtQU1Nj+MQb/UUEPeEMmMlsWyGN1mnUWri0f/9+xa6urg57n5HAJ3BCCHEUBnBCCHEUBnBCCHEU352B642a9HMnwJwaoxcmAMD06dMV+6677jJ8tm5Ve8W7MkhXb7wDADNnzlTsHTt2GD5jxoxRbFuRR2tra5S78wb9bLW2ttbw0c+zbQ2m9EZVPW3EnZ7TAMymTPoZMQAcOnSoWx/bmbcf0e8VW2MuPYZ88MEHho9+5p+fn2/4bNmyJZItRgyfwAkhxFEYwAkhxFEYwAkhxFEYwAkhxFF8l8TUsU2f0ROb11xzjeGjd5XTk1WA2WFtyJAhhs8bb7wR1j79xqpVq4y1t956S7GXLl1q+GRkZCi2bdqRH7EVzugTh2zFGV9++aViL1myxPDRCzZ0jVxD735nu+/1jpO2BL/eldFWIONHbPeKHlMCw8kV9E6D+mQdwIw7tmI5WyyKFD6BE0KIozCAE0KIozCAE0KIo/j+DDxSHnjgAcV+7LHHDB99TX9RHwB2796t2PpEb79ia0I1bdo0xdYb+ADAyy+/rNi5ubmGj96wx6/oZ536RCIAuPvuuxXbdva5ePFixX799dcNH1szLVewFfuMHz9esfXCHsA8Jy8uLjZ89OkzfkXPtYVTAGVrVKXnjAoLCw2f48ePK7atsChc+AROCCGOwgBOCCGOwgBOCCGOwgBOCCGO0iOSmGfPnjXWrr/+esXWExAAcPjwYcXesGGD4eNK0jIc9I5yjzzyiOGjd238+OOPDR99clFLS0sMdhd/bF0r9Qk2v/zyi+GjTzuydcjUtU3URJ5YoU/p0QueALNjn54AB8xJPrZJUX7E9jPWpxTZunk+/fTTiq0Xk3X1dZHCJ3BCCHEUBnBCCHEUBnBCCHEUJ8/A9Zfs9aIDADhz5oxi//DDD4aP3tTnnXfeMXxshSx+RJ/SfuzYMcNHP8vdt2+f4XPHHXcodt++fQ0fV8689XPoAwcOGD5vvvmmYus6AmaByqhRowyfWJ5rxhs9F2BrVKWvTZw40fDRm6PZcip6IZxf0Qvf0tPTDZ+TJ08qtq2QR/9M5eXlGT48AyeEEMIATgghrsIATgghjtJtABdCbBBCnBBCVISspQkhSoUQVYFfe9Yo7zD47rvv8Omnn2Lnzp3Btfb2dnzzzTcAMCYZdXnppZeQn5+vNAVrbm5GUVERkKSaPPPMM8jNzcX06dODa01NTXjooYeAJNXk1VdfRWFhoTJQpaWlBc899xxqamqQjJpESjhJzGIArwMIzfAtA7BDSvlvIcSygP1cLDY0depUxc7MzDR82traFNuWoNSnXtgKciZMmKDYF5KwzMnJwbBhw5TEWHV1NTIyMlBfX18BYAdipItebPLHH38YPnqC0paYys7OVmy9YyNgJl3OnTsX9j7vuecezJ49W5lgU1xcjAkTJmDPnj0x1UTvHhf6F2knX3zxhWLv3bvX8NELL2xJJ70YxVY41hWFhYWYN2+ecp1169ZhypQp2L17d0w1AcwE2WeffWb4fPLJJ4pt+4ytWLFCsW+77TbDRwih2AcPHjR8bBO1br/9dhQUFGD16tXBtffffx/jxo1Da2srqqqqYqqJPnFo+PDhhs+RI0cUu7Ky0vBpbW1V7ClTphg+ffr0+cfvC9g1iZRun8CllLsAnNKWCwBsDPx+I4D/i9mOHCE9Pd34YR0/fhw5OTmdZtLpkpubi9TUVGVt586dmDlzZqeZdJpMmjTJ+AugtLQU9913X6eZdJqMHTvWGMf21Vdfhf4rJek0iZRIz8CzpJR1ABD4tctmyEKIRUKIMiFEme0VrZ7EmTNn0K9fPwD/rEuoJrYe1T2JhoaG4BNeuJqcOqU/L/Qs6uvrkZWVBeDCPj/Nzc1ebdFzGhsbg6/uMaaET9yTmFLK9VLK8VLK8SkpKfG+nBOEajJ4MI/6AFUTvddKMhOqi/6vm2SFMeVvIi3kOS6EuEJKWSeEuALAiW6/woJeSAOYL8vbmgt99NFH3X4ffUK2rRAh1vTt2zfYBCdSXWwTq48eParYpaWlho9enDFr1izDJzSRBpgNmOJBenp68GcaqSa2wLVmzZpuv07/mS9cuNDwCTnyAmA/n9QLgqJtVJWRkRGcyhLN50dvrgSY989NN91k+OifDduZsD5N3dYILJb/Uho8eHBwMk00mlRUVBhremOubdu2GT56cZat+Z2ugT6RCOhI2ntJpJ/grQDmBX4/D4DZsi4JycrKQm1tbadJXQDccsstoQlVagIgPz8fW7Zs6TSpCYDJkyeHPphQkzDp9glcCLEJwG0AMoQQtQBeBPBvAJuFEAsBHAVgDn7r4ezfvx8NDQ1ob2/H9u3bMXLkSIwYMaKzlHYMgGYkmS7Lly9HWVkZmpqacOedd2Lx4sWYP38+li1bBiSpJkuWLMHXX3+NxsZGTJo0CUuXLkVRUVHw1UokoSarVq1CeXk5mpubMXfuXDz66KOYM2cOVq5ciZqaGgCYjiTTJFK6DeBSSnPybQfTulhPCrp65XDy5MkoKSmpkFImnT62ftBAR7+RvLy8pNRk7dq11vVNmzZh6NChSanJCy+8YF1fs2YNioqKUFlZmXSaRAorMQkhxFES2o1w9OjRxlo4E3DGjBkTj+34Apsm+tqMGTO82o4v0N+3B4BFixZd8PfRC08AtyfnBI4bFPSuejZ0HTpffQ1FT5y7gq04TU/k6oVxgFlwYysOrKuri3J3sYdP4IQQ4igM4IQQ4igM4IQQ4igilo1Vur2YECcBHAGQAaC+G3e/cSF7HiqlNDsEWaAmJiGaXOg1/EBcNAGcvleoiZ2odfE0gAcvKkSZlNKcg+Zj4r1napK4a8QSamJCTezEYs88QiGEEEdhACeEEEdJVABfn6DrRkO890xNEneNWEJNTKiJnaj3nJAzcEIIIdHDIxRCCHEUBnBCCHEUzwO4EGKGEOKQEKI6MBDZVwghNgghTgghKkLW0gKTsqviMTHb75oA3utCTazXoyb2a/pal3hq4mkAF0L0BvAfAHcCGA1grhDC7N6UWIoB6N2ilgHYIaW8Fn9PEY8JjmgCeKgLNTGhJnYc0aUYcdLE6yfwiQCqpZSHpZTtAP6Ljgn3vkFKuQuAPiuqAB2TsoHYT8z2vSaA57pQExNqYsf3usRTE68D+JUAQodc1gbW/E5WYFJ2txOzI8BVTYD46UJNTKiJHVd1iYkmXgdwsyEzkOzvMVITE2piQk3sJLUuXgfwWgBDQuwcAMc83kMkHA9Myo5qYnYXuKoJED9dqIkJNbHjqi4x0cTrAP4tgGuFEMOFECkA5qBjwr3f2YqOSdlA7Cdmu6oJED9dqIkJNbHjqi6x0URK6el/AO4CUAngJwAveH39MPa3CUAdgLPo+Nt9IYB0dGSKqwK/piWTJonQhZpQk56iSzw1YSk9IYQ4CisxCSHEURjACSHEURjACSHEURjACSHEURjACSHEURjACSHEURjACSHEUf4HuTWF6miRI9AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pool = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=(2,2), padding='same')(conv2d)\n",
    "print(pool.shape)  # (1, 13,13, 5)\n",
    "feature_maps = np.swapaxes(pool, 0, 3)\n",
    "for i, feature_map in enumerate(feature_maps):\n",
    "    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(13,13), cmap='gray')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}