GD_AGD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b06a418",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cvxpy as cvx\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "70803da0",
   "metadata": {},
   "outputs": [],
   "source": [
    "ITER = 200000\n",
    "lr = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "5caf2f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "def AGD(x, A):\n",
    "    z, norm_lst = x, []\n",
    "    lamb, gamma = 1, 0\n",
    "    for i in range(10000):\n",
    "        norm_lst.append(np.linalg.norm(A @ x) ** 2 + 1e-6)\n",
    "        if i > 3:\n",
    "            lamb, gamma = 6 / (i * (i - 1)), 2 / i\n",
    "        y = lr * ((1 - gamma) * x + gamma * z)\n",
    "        f_grad = 2 * A.T @ A @ y\n",
    "        z = lr * (z - (gamma / lamb) * f_grad)\n",
    "        x = y - f_grad\n",
    "    return x, norm_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "aab22498",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 20)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = np.random.normal(0, 1, (20, 20))\n",
    "A.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3fe6fd48",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_AGD, AGD_norm_lst = AGD(np.ones((20,)), A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b0a28343",
   "metadata": {},
   "outputs": [],
   "source": [
    "def GD(x, A):\n",
    "    norm_lst = []\n",
    "    for _ in range(10000):\n",
    "        norm_lst.append(np.linalg.norm(A @ x) ** 2)\n",
    "        x = x - 1e-3 * 2 * A.T @ A @ x\n",
    "    return x, norm_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "2807c7b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_GD, GD_norm_lst = GD(np.ones((20,)), A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "c80ed415",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'loglog')"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEbCAYAAADERMP2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAoYklEQVR4nO3deXxV9Z3/8dcne4CQAEmQ1bAvSkUbse4sLtQRcWyrtp2pVkfGTu2vtY+OdTodtdvYZTptHZ222FrqryPoqHVQ27GtgOBaFlGkYVckgGRBtsSQkHzmj3sTLzG5uUnO5d7c+34+Hnnknu8553s+OQ/IJ9/v95zv19wdERGRzmQkOgAREUluShQiIhKVEoWIiESlRCEiIlEpUYiISFRKFCIiEpUShUiMzOwtM7uol3UsMrNvBxWTyImgRCEiIlEpUYiISFRKFCLdZGa5ZvZjM9sT/vqxmeVG7L/NzPaG9/2dmbmZje+krpvMbJuZ7TezpWY2PGLfJWa22cwOmtl/mtlzZvZ3J+JnFImkRCHSff8MfASYDpwGzAC+DmBmc4EvAxcB44ELO6vEzGYDdwNXA8OAncCS8L5i4FHgn4AhwGbgnHj8MCJdUaIQ6b5PA9909yp3rwa+AfxteN/VwK/cfaO714f3RavnAXdf5+5HCSWFs82sDLgM2Ojuj7v7MeAe4J04/TwiUSlRiHTfcEJ//bfaGS5r3bcrYl/k56j1uPsRoBYY0b4eD83eWdmrqEV6SIlCpPv2ACdHbI8OlwHsBUZG7BsVaz1m1p9QN9Pu9vWYmbWrV+SEUaIQ6b7FwNfNrCQ8lnAH8JvwvkeAz5rZFDPrF97XmYfCx04PD4b/K/CKu78FPA1MM7MrzSwL+DxwUpx+HpGolChEuu/bwBrgdWADsC5chrv/ntB4wnJgG/BS+Jyj7Stx92eBfwEeI9SCGAdcG95XA3wC+D6h7qip4Wt+oB6ReDMtXCQSP2Y2BXgDyA0PSve0ngxCYxSfdvflQcUnEgu1KEQCZmZ/bWY5ZjYI+B7wZE+ShJldamZF4W6prwEGvBxwuCJdUqIQCd7fA9XAdqAZ+FwP6zk7XEcNMA+40t3fCyRCkW5Q15OIiESlFoWIiESlRCEiIlFlJTqAIJnZPGBeQUHBTRMnTkx0OCIifcratWtr3L2kfXlKjlGUl5f7mjVrEh2GiEifYmZr3b28fbm6nkREJColChERiSqlEoWZzTOzhQcPHkx0KCIiKSOlBrPd/UngyfLy8psSHYuI9D1NTU1UVlbS0NCQ6FDiKi8vj5EjR5KdnR3T8SmVKEREeqOyspKCggLKysoIzeyeetyd2tpaKisrGTNmTEznpFTXk4hIbzQ0NDBkyJCUTRIAZsaQIUO61WpSohARiZDKSaJVd3/GlOx62rjnENPufKZnJxtkZRiZ4a+sjIy2z6Ft63A7KyODjOO27QPbmRkZZGbQVmdHx2REHNtanpuVQW52Zuh7Vga5WZnkhD/nZYe2W8tzszPIyQzFIiJ90759+7j11lt5+eWXGTRoEDk5Odx2220MGjSI+fPnM3bsWOrr6xk6dCi33XYbl19+eVzjSalE0fpm9pBR4/lEebQVKDvX4k6LO8danJaW0Pfmlsjtlrbt5oh97zU1tzun5bhj2p/T3K6uoN97zMkMJ5VwIumfm0m/nCz652bSPyeL/rnHf+6Xkxkuy6J/6+ecLPrlZjIwL5vC/GxystQAFYk3d+fKK6/kuuuu46GHHgJg586dLF26lEGDBnH++efz1FNPAbB+/XquvPJK8vPzmTNnTtxiSqlEEfnU0x3zpiY6nG5paXGa/YMJ5VhzC0ePtX41h743hT43Hmu3r+n44xrDn99rbKG+8Rh1jc3UHT1G7ZF66hqPUX+0mbrGYzQ0tcQUY7+cTIrysxmYn01Rv1DyKMrPobD1c79sigfkUjwgJ/w9l/65KfVPTCTuli1bRk5ODjfffHNb2cknn8wXvvAFVqxYcdyx06dP54477uDee+9VokgHGRlGBkZ25om/9rHmFuqbQkmk7mj4e+P7nw83NHGgvomD7zVx4L3Q94P1TbxZU8fB9w5woL6Jo8c6Tjb52ZkUF7yfOIoH5FIyIIehhXkML8xnWFEewwrzGZiXlRZ9w9J3fOPJjfxlz6FA65w6fCB3zjsl6jEbN27kjDPOiLnOM844gx/84Ae9DS0qJQohKzODgZkZDMyL7ZnqjjQ0NfNufSO1RxqpPnKUmsNHqTnSSM2Ro9QcOUrtkUZ27a/n1bffpbau8QNdbf1zMhlWlM+wiAQyoiifsuL+nDy4HyUFuUokkpY+//nP8/zzz5OTk9NhQjgR8/UpUUgg8rIzGVaYz7DC/C6PPdbcQtXho+w9+B57DjQc933vwQYq9h6m5sjR487Jz87k5CH9GD24H2XF/UPfh/RnbEl/hhXmKYlI4Lr6yz9eTjnlFB577LG27fvuu4+amhrKyz8wVx8Ar776KlOmTIlrTEmfKMysP/CfQCOwwt3/K8EhSS9lZWYwvCif4UX5fPjkjo85eqyZPQca2Flbx9v763mrpp6399exo6aOFVuqaYzo6hqQm8X40gFMKB3AhKEDmFBawPjSAYwoytfTX9LnzJ49m6997Wv89Kc/5XOfC62iW19f3+Gxr7/+Ot/61rf4xS9+EdeYEpIozOwB4HKgyt1PjSifC/wEyAR+4e7fBa4CHnX3J83sYUCJIg3kZmUyprg/Y4r7f2BfS4vzzqEG3qqtY3t1Hdv2HWZr1RFWbKnmv9dWth2Xn53J+NIBTBlWwNRhAzllRCGTTyqgoBddbCLxZmY88cQT3HrrrXz/+9+npKSE/v37873vfQ+AVatWcfrpp1NfX09paSn33HNPXAeyIXEtikXAvcCDrQVmlgncB1wMVAKrzWwpMBLYED6s+cSGKckoI8PaWiTnjCs+bt+B+ka2VR1ha9URtu47wpZ9h3m2oopH1ryfQMqG9OOU4YVMHT6QqcMHcurwQkoKck/0jyHSqWHDhrFkyZIO9yVi0tOEJAp3X2lmZe2KZwDb3H0HgJktAeYTShojgfXoTXLpQlG/HMrLBlNeNritzN3Zd+gof9l7kI27D/GXvYfYsPsgT2/Y23bMiKJ8po8qCn2NLuLU4YXk5yTgETSRJJRMYxQjgF0R25XAWcA9wL1m9lfAk52dbGYLgAUAo0ePjmOY0teYGScV5nFSYR6zJw9tKz/U0ETFnlDSeHXXAda/faAteWRmGJNPKmD6qCJOG1XEGaOLGFcyQIPmkpaSKVF09D/Q3b0O+GxXJ7v7QjPbC8zLycn5cODRScoZmJfNWWOHcNbYIW1l1YePsn7XAdbvepf1uw6wdP0e/uuVtwEY1C+b8rLBzCgbzJljBnPK8IFkZ6qRK6kvmRJFJRA578ZIYE93KtB6FNJbJQW5XDx1KBdPDbU8Wlqc7dVHWPf2u6x+611Wv7WfP/5lHxAaLD99dBFnlg3mzLLBnD66SG+iS0pKpn/Vq4EJZjYG2A1cC3yqOxW0zvU0fvz4OIQn6Sgjw5gwtIAJQwu45sxQl2bVoYa2pPHnN/dzz7KtuIe6q04dPpCPjBvCOeOKObNsEP1ykum/mEjP2Il4q+8DFzVbDMwEioF9wJ3u/kszuwz4MaHHYx9w9+/0pP7y8nJfs2ZNQNGKRHeooYl1O99PHOt3HaCp2cnKMKaPKuKccUM4e1wxp48uIi8Rc7RIzCoqKuL+8lqy6OhnNbO17v6BN/sSkijiJaJFcdPWrVsTHY6kqfrGY6x5611e2lHLi9tr2VB5gBaH3KwMyssGcfbYUOL40MhCjXEkmWRKFL/97W+56qqrqKioYPLkyQD8+c9/5rbbbmP37t0UFBQwbNgwvvvd7zJt2jTuuusu7r//fkpKSqirq2PatGl8+9vfZurUjidI7U6iSKl2scYoJBn0y8nigoklXDCxBAi1OP68Y39b4vi3P2wBttA/J5MZYwZz7vhizp9QwsSheqpK3rd48WLOO+88lixZwl133cW+ffu4+uqreeihhzjnnHMAeP7559m+fTvTpk0D4NZbb+UrX/kKAA8//DCzZ89mw4YNlJSU9CqWlEoUGqOQZDQwL5uLpg7lovAA+f66Rl7eUctL22t5YXsNy5+uACooLcjlvAnFnD+hmPPGl+glwDR25MgRXnjhBZYvX84VV1zBXXfdxb333st1113XliQAzjvvvE7ruOaaa3j66ad56KGH+OIXv9ireFIqUahFIX3B4P45XDZtGJdNGwbAngPv8fzWGlZtq2H5pioeX7cbgCnDBnJ+OHGcWTZY4xsn2u9vh3c2dH1cd5w0DT763S4Pe+KJJ5g7dy4TJ05k8ODBrFu3jo0bN3Ldddd163JnnHEGmzZt6mm0bVIqUahFIX3R8KJ8rj5zFFefOYqWFmfjnkOs2lbNqi01LHrhLRau3EFuVgYzxgwOJ44SJp9UoG6qFLZ48WK+9KUvAXDttdeyePHiDxxz1llncejQIS655BJ+8pOfdFhPUGPQKZUo1KKQvi4jw5g2spBpIwv5h5njqW88xitv7mfVlhpWba3mX3+3CdhE8YDccBdVMRdMVDdVXMTwl3881NbWsmzZMt544w3MjObmZsyM6667jnXr1jF//nwAXnnlFR599NG2ZVE78uqrr3Y6PXl3pFSiEEk1/XKymDWplFmTSgF452ADq7ZWs2prDSu3VPPbV0PdVKeOGMiFE0uYOamU00cVkaWnqfqsRx99lM985jP8/Oc/byu78MILueSSS/ibv/kbLr300rZxis6mHwd47LHH+MMf/sAPf/jDXseUUolCXU+S6k4qzOMT5aP4RHmom+ovew/x3JZqVmyu4mfP7eC+5dspyMvi/AnFzJxYyoWTShg6MC/RYUs3LF68mNtvv/24so997GM89NBDPPzww3z1q19l9+7dlJaWUlxczB133NF23I9+9CN+85vfUFdXx6mnnsqyZct6/cQTpNh7FK30wp2ko4PvNfHCthqe21zNii1V7DsUWiVw8kkFzJxUysxJJXz45EF6dyOKZHqPIt7S9j0KkXRWmJ/d9jSVu7PpncNtrY1frNrBz57bzoDcLM4dP4SZk0q5cGIJw4u6XrpWRIlCJAWZGVOGDWTKsIHcfOE4Djc08eL2WlZsrua5zVU8szE0seHEoQPakkZ52SBys/QIrnxQSiUKjVGIdKwgL5tLTzmJS085CXdnW9URVoS7qFofwe2Xk8k544qZOamEWZNLGaHWhoSlVKLQ47EiXTN7f0bcmy4YS93RY7y0vZYVW6pYsbmaP1WEWhuTTypg1uRSZk9Oryep3D3l31Hp7th0SiUKEem+/rlZbVOMuIfW31i2qYplm6q4f+UOfrpiO4X52Vw4sYTZk0PdVIP65yQ67LjIy8ujtraWIUOGpGyycHdqa2vJy4v9aTg99SQinTrU0MSqLTUs21TFc1uqqDnSSIbB6aMHMXty6P2OKcNS5y3xpqYmKisraWhoSHQocZWXl8fIkSPJzs4+rjwtphlvpUQhEryWFuf13QdZtqmK5Zuq2LD7IADDCvOYOSnURXXu+CFarKkPU6IQkUBVHWpgxeZqlm2qYtXWauoam8nJyuAjY4cwe1IJsycPZfSQfokOU7ohLRKFFi4SSYzGYy2sfmt/W2tjR00dAONK+oe6qCaXcmbZYL3sl+TSIlG0UotCJLHeqqkLJY3NVbyyYz+NzS0U5GZx/sRiZk0qZeakUk1kmISUKEQkIeqOHuP58FobyzeHphYxgw+NKGT25KHMmVLKKcMHpsyAeF+mRCEiCeceWm9j+aYqlm2uYv2uA7iHBsRnTy7loilDOXvcEC3SlCBKFCKSdGqOHGX5piqerahi5dZq6hubyc/O5LwJxcyZXMrsKaWUFmj22xNFiUJEktrRY828vGM/z1bs49mKKnYfeA+A00YWMmdKqItq6jB1UcVTn00UZjYW+Geg0N0/Hss5ShQifVvr7LfPVuzjTxVVvFYZ6qIaXpjH7CmlzJkylLPHqosqaAlJFGb2AHA5UOXup0aUzwV+AmQCv3D3LtccNLNHlShE0lP14VAX1Z8q9rFqaw3vNTXTLyeT88YXc9GUocyarKeogpCoRHEBcAR4sDVRmFkmsAW4GKgEVgOfJJQ07m5XxQ3uXhU+T4lCRGhoaualHbVtXVR7DzZgBqeNLOKicGtj8kmpM63IiZSwriczKwOeikgUZwN3uful4e1/AnD39kmifT1RE4WZLQAWAIwePfrDO3fuDOYHEJGk5R5aDvbZiiqerdjHa5WhaUVGFOUzJ5w0PjJ2sNbZiFEyJYqPA3Pd/e/C238LnOXut3Ry/hDgO4RaIL/oKqGAWhQi6arqUAPLNlXxp4oqnt9WTUNTC/1zMjl/QglzpoTeEC8eoC6qziTTUqgdtQc7zVbuXgvcHFPFWrhIJK2VDszj2hmjuXbGaBqamnlxew1/qqhiWUUV/7vxHcxg+qgiLpoylNmTS9VFFaNEJIpKYFTE9khgTwLiEJEUlpedyezJQ5k9eSh+ZehFv2crqli2aR8/eGYzP3hmMyOK8pk9uZQ5U0r5iJ6i6lQiup6yCA1mzwF2ExrM/pS7bwzqmup6EpFoqg41sHxzuIuq3VNUrV1U6fiiX6KeeloMzASKgX3Ane7+SzO7DPgxoSedHnD37wR0Pc0eKyLdEvkU1bKKKvYcDC1adNqoIuaEWxvp8qJfn33hrifUohCRnnB3KvYeZtmm41/0a52Las6UUs4eW0x+Tmp2UaVFolCLQkSCVH34KMs3hx69XbW1hvrGZnKzMjh73JC2pWBHDU6dxZnSIlG0UotCRIJ29Fgzr+zYz/LNocWZ3qqtByIWZ5pUSnnZYHKy+u7iTEoUIiIBerOmrm2NjdbFmQbkZnHe+GJmTS5h5qRShg7sWwPiaZEo1PUkIolQd/QYL26vZdmmKlZsDk0rAnDK8IHMmhR6imr6qCIyM5J7QDwtEkUrtShEJFHcnc37DoeSxqZq1r79Ls0tTlG/bC6cWMLsyaVcMKGEQf1zEh3qByhRiIgkwMH6JlZurWb55iqe21xNbV0jGeE3xGdPDq0fnixLwaZFolDXk4gks5YW5/XdB9vGNl4PT2JYWpAb7qIq4dzxxRTkZSckvrRIFK3UohCRvqD68FFWbK5ixeZqVm6p5vDRY2RnGmeWDWbWpFIu+9AwRhTln7B4lChERJJYU3MLa3e+2/b47ZZ9RwCYUTaY+acP56+mDaOoX3zHNZQoRET6kJ21dSxdv4cn1u9me3Ud2ZnGhRNLmT99OBdPHRqXCQzTIlFojEJEUo17aObbJ17dzdLX9lB1+ChF/bL569NH8MkZo5k4tCCwa6VFomilFoWIpKLmFufF7TUsWb2LP2x8h6Zm54zRRVw7YzSXf2gY/XJ6t3KEEoWISAqpPXKUx9ftZvHqt9lRXUdBbhZXnTGCmy4Yy8hBPZt/qrNE0XcnJRERSWNDBuRy0wVjefbLF/LI35/NnCmlLP7zLg7UNwV+rUSscCciIgExM2aMGcyMMYO564rGuDwZlVItCjObZ2YLDx48mOhQREROuHg9PptSicLdn3T3BYWFhYkORUQkZaRUohARkeApUYiISFRKFCIiEpUShYiIRNUnEoWZXWlm95vZ/5jZJYmOR0QknXSZKMysn5n9i5ndH96eYGaXx3oBM3vAzKrM7I125XPNbLOZbTOz26PV4e5PuPtNwPXANbFeW0REei+WFsWvgKPA2eHtSuDb3bjGImBuZIGZZQL3AR8FpgKfNLOpZjbNzJ5q91UacerXw+eJiMgJEsub2ePc/Roz+ySAu79n3Vizz91XmllZu+IZwDZ33wFgZkuA+e5+N/CB1kr4et8Ffu/u62K9toiI9F4sLYpGM8sHHMDMxhFqYfTGCGBXxHZluKwzXwAuAj5uZjd3dICZLTCzNWa2prq6upfhiYhIq1haFHcC/wuMMrP/As4lNFbQGx21SDqdxtbd7wHuiVahuy80s73AvJycnA/3Mj4REQmL2qIwswxgEHAVoeSwGCh39xW9vG4lMCpieySwp5d1agoPEZE4iJoo3L0FuMXda939aXd/yt1rArjuamCCmY0xsxzgWmBpbyvVpIAiIsGLZYzij2b2FTMbZWaDW79ivYCZLQZeAiaZWaWZ3ejux4BbgGeACuARd9/Yo58ggloUIiLB63KFOzN7s4Nid/ex8Qmp57RmtohIz2kpVBERiaqzRBHTCndmdiqhF+PyWsvc/cHgwgtGRIsi0aGIiKSMWKbwuBP4j/DXLOD7wBVxjqtHNEYhIhK8WAazPw7MAd5x988CpwG5cY2qh/TUk4hI8GJJFO+FH5M9ZmYDgSog6QayQS0KEZF4iGWMYo2ZFQH3A2uBI8Cf4xmUiIgkjy4Thbv/Q/jjz8zsf4GB7v56fMPqGQ1mi4gEL5bB7Atav4DRQFH4c9JR15OISPBi6Xr6x4jPeYSmCF8LzI5LRCIiklRi6XqaF7ltZqMIPSIrIiJpoCdrZlcCpwYdSBD0eKyISPC6bFGY2X/w/loRGcB04LU4xtRj7v4k8GR5eflNiY5FRCRVxPR4bMTnY8Bid38hTvGIiEiSiWWM4tcnIhAREUlOsXQ9baDjZUqN0HTjHwo8KhERSRqxdD39Pvz9/4e/fxqoB9TSEBFJA7EkinPd/dyI7dvN7AV3/2a8guopvZktIhK8WB6P7W9m57VumNk5QP/4hdRzejNbRCR4sbQobgQeMLPW374HgBviFpGIiCSVWJ56WgucFp5i3Nxdb7OJiKSRWCYF/GI4SRwGfmhm68zskviHJiIiySCWMYob3P0QcAlQCnwW+G5coxIRkaQRS6Kw8PfLgF+5+2sRZXFnZlPM7Gdm9qiZfe5EXVdEREJiSRRrzewPhBLFM2ZWALTEUrmZPWBmVWb2RrvyuWa22cy2mdnt0epw9wp3vxm4GiiP5boiIhKcWJ96mg7scPd6MxtCqPspFouAe4EHWwvMLBO4D7iY0Ey0q81sKZAJ3N3u/BvcvcrMrgBuD9clIiInUCxPPbUA6yK2a4HaWCp395VmVtaueAawzd13AJjZEmC+u98NXN5JPUuBpWb2NPBQR8eY2QJgAcDo0aNjCU9ERGIQS4siaCOAXRHblcBZnR1sZjOBq4Bc4HedHefuC4GFAOXl5R3NTSUiIj2QiETR0UB4p7/Y3X0FsCKmijWFh4hI4GJKFGY2CBgVeby7r+v8jKgqw3W1Ggns6WFdIiISZ7FMM/4t4HpgO+//5e/A7B5eczUwwczGALuBa4FP9bCu42iFOxGR4MXSorgaGOfujd2t3MwWAzOBYjOrBO5091+a2S3AM4SedHrA3Td2t+5OrqeuJxGRgJl79HFfM3sM+Jy7V52YkHqvvLzc16xZ0/WBIiLSxszWuvsH3leLpUVxN/Bq+KW5o62F7n5FgPEFQi0KEZHgxZIofg18D9hAjG9kJ4rGKEREghdLoqhx93viHomIiCSlWBLFWjO7G1jK8V1PPX08Nm7U9SQiErxYBrOXd1Ds7t7Tx2PjToPZIiLd16PB7PAEfkvd/Udxi0xERJJa1GnG3b0ZSLqnmzpjZvPMbOHBg1qtVUQkKLGsR/Gimd1rZueb2RmtX3GPrAfc/Ul3X1BYWJjoUEREUkYsg9nnhL9/M6KsN1N4iIhIHxLLehSzTkQgQdBTTyIiweuy68nMCs3s381sTfjrh2aWlH076noSEQleLGMUDwCHCU0OeDVwCPhVPIMSEZHkEcsYxTh3/1jE9jfMbH2c4hERkSQTS4viPTM7r3XDzM4F3otfSCIikkxiaVHcDDwYHpcwYD+hhYxERCQNxPLU02vAaWY2MLx9KO5R9ZCeehIRCV4sS6HmAh8DyoAsMwPA3b8Z5bSE0DTjIiLBi6Xr6X+Ag8BaImaPFRGR9BBLohjp7nPjHomIiCSlWOd6mhb3SEREJCnF0qI4D7jezN4k1PVkhNaj+FBcIxMRkaQQS6L4aNyj6IKZ9QdWAne6+1OJjkdEJJ102fXk7js7+oqlcjN7wMyqzOyNduVzzWyzmW0zs9tjqOqrwCOxXFNERIIVS4uiNxYB9wIPthaEV827D7gYqARWm9lSIBO4u935NwAfAv4C5MU5VhER6UBcE4W7rzSzsnbFM4Bt7r4DwMyWAPPd/W7g8vZ1mNksoD8wldB0Ir9z95YOjlsALAAYPXp0oD+HiEg6i3eLoiMjgF0R25XAWZ0d7O7/DGBm1wM1HSWJ8HELgYUA5eXlHlSwIiLpLhGJwjoo6/IXu7sv6rJiTeEhIhK4WN6jCFolMCpieySwJ4iKtXCRiEjwEpEoVgMTzGyMmeUA1wJLg6jYzOaZ2cKDBw8GUZ2IiBDnRGFmi4GXgElmVmlmN7r7MeAW4BmgAnjE3TfGMw4REek5c0+9cd/y8nJfs2ZNosMQEelTzGytu5e3L09E11PcqOtJRCR4KZUoNJgtIhK8lEoUIiISvJRKFOp6EhEJXkolCnU9iYgEL6UShYiIBC+lEoW6nkREgpdSiUJdTyIiwUupRCEiIsFTohARkahSKlFojEJEJHgplSg0RiEiEryUShQiIhI8JQoREYlKiUJERKJKqUShwWwRkeClVKLQYLaISPBSKlGIiEjwlChERCQqJQoREYlKiUJERKJK+kRhZjPNbJWZ/czMZiY6HhGRdBPXRGFmD5hZlZm90a58rpltNrNtZnZ7F9U4cATIAyrjFauIiHQsK871LwLuBR5sLTCzTOA+4GJCv/hXm9lSIBO4u935NwCr3P05MxsK/Dvw6TjHLCIiEeKaKNx9pZmVtSueAWxz9x0AZrYEmO/udwOXR6nuXSA3LoGKiEin4t2i6MgIYFfEdiVwVmcHm9lVwKVAEaHWSWfHLQAWAIwePTqIOEVEhMQkCuugzDs72N0fBx7vqlJ3X2hme4F5OTk5H+5FfCIiEiERTz1VAqMitkcCe4KoWFN4iIgELxGJYjUwwczGmFkOcC2wNIiKNSmgiEjw4v147GLgJWCSmVWa2Y3ufgy4BXgGqAAecfeNQVxPLQoRkeDF+6mnT3ZS/jvgd0Ffz8zmAfPGjx8fdNUiImkr6d/M7g61KEREgpdSiUJjFCIiwUupRKEWhYhI8FIqUahFISISvJRKFGpRiIgEL6UShYiIBC+lEoW6nkREgpdSiUJdTyIiwUupRCEiIsFTohARkahSKlFojEJEJHgplSg0RiEiEryUShQiIhI8JQoREYlKiUJERKJSohARkahSKlHoqScRkeClVKLQU08iIsFLqUQhIiLBU6IQEZGolChERCQqJQoREYkqK9EBdMXMMoBvAQOBNe7+6wSHJCKSVuLaojCzB8ysyszeaFc+18w2m9k2M7u9i2rmAyOAJqAyXrGKiEjH4t2iWATcCzzYWmBmmcB9wMWEfvGvNrOlQCZwd7vzbwAmAS+5+8/N7FHg2S6vemAn/Pbm1iu2XrjdduQJnR1jCTqGjo8REelK+Q0weEygVcY1Ubj7SjMra1c8A9jm7jsAzGwJMN/d7wYub1+HmVUCjeHN5s6uZWYLgAUA04fnwM4XwNsiaQ3o+O2OyuJ6TBfxeMSxHZaJiHRh0mV9K1F0YgSwK2K7EjgryvGPA/9hZucDKzs7yN0XAgsBysvLnS+tCSBUERFJRKLoqB+l0z+b3b0euDGmis3mAfPGjx/fw9BERKS9RDweWwmMitgeCexJQBwiIhKDRCSK1cAEMxtjZjnAtcDSICrWXE8iIsGL9+Oxi4GXgElmVmlmN7r7MeAW4BmgAnjE3TcGdD3NHisiEjDzFHyqpry83Nes0WC2iEh3mNlady9vX55SU3ioRSEiEryUShQaoxARCV5KJQoREQle0k8K2B2t71EAh8xsa7i4EGjfF9W+LHK7GKiJU4gdxRLUOdGO62xfLPemo7Jkvl+xnhfU/eqoXPcr+r50v1/R9if6fp3cYam7p/QXsLCrsshtQjPUnrBYgjon2nGd7Yvl3vS1+xXreUHdr67uTzrfr872pfv9irY/We9XOnQ9PRlDWUfHxENPrhPrOdGO62xfLPemo7Jkvl+xnhfU/eqoXPcr+r50v1/R9ifl/UrJx2N7w8zWeAePh0nHdL+6R/ere3S/uide9ysdWhTdtTDRAfQxul/do/vVPbpf3ROX+6UWhYiIRKUWhYiIRKVEISIiUSlRiIhIVEoUXTCz/mb2azO738w+neh4kp2ZjTWzX4bXN5cumNmV4X9b/2NmlyQ6nmRnZlPM7Gdm9qiZfS7R8fQF4d9ha83sA0tNxyotE4WZPWBmVWb2RrvyuWa22cy2mdnt4eKrgEfd/SbgihMebBLozv1y9x3uHtOKhKmqm/frifC/reuBaxIQbsJ1835VuPvNwNVAWj42283fXwBfBR7pzTXTMlEAi4C5kQVmlgncB3wUmAp80symElqBr3WN7+YTGGMyWUTs90t6dr++Ht6fjhbRjftlZlcAzwPPntgwk8YiYrxfZnYR8BdgX28umJaJwt1XAvvbFc8AtoX/Im4ElgDzCS3dOjJ8jO7X+zq7X2mvO/fLQr4H/N7d153oWJNBd/99uftSdz8HSMuu4G7er1nAR4BPATeZWY9+h6XUpIC9NIL3Ww4QShBnAfcA95rZX3HiphboCzq8X2Y2BPgOcLqZ/ZO7352Q6JJPZ/++vgBcBBSa2Xh3/1kigktCnf37mkmoOzgX+N2JDytpdXi/3P0WADO7Hqhx95aeVK5E8T7roMzdvQ747IkOpg/o7H7VAjef6GD6gM7u1z2E/hiR43V2v1YAK05sKH1Ch/er7YP7ot5UnpZdKZ2oBEZFbI8E9iQolr5A96t7dL+6R/ere+J6v5Qo3rcamGBmY8wsB7gWWJrgmJKZ7lf36H51j+5X98T1fqVlojCzxcBLwCQzqzSzG939GHAL8AxQATzi7hsTGWey0P3qHt2v7tH96p5E3C9NCigiIlGlZYtCRERip0QhIiJRKVGIiEhUShQiIhKVEoWIiESlRCEiIlEpUYiISFRKFCIBM7Nvhqd3xsy+ZGb9Aqz7ysjpySOvJRIveuFOJI7M7C2g3N1runFOprt3uPaJmS0CnnJ3rSAoJ4xaFJIWzKzMzCrCy45uNLM/mFm+ma0ws/LwMcXhX+yY2fVm9oSZPWlmb5rZLWb2ZTN71cxeNrPBUa61yMw+bmb/DxgOLDez5eF9l5jZS2a2zsz+28wGhMvfMrM7zOx54BNmdpOZrTaz18zsMTPrZ2bnEFpl8Qdmtt7MxrVeK1zHnHB8G8KroOVG1P2N8DU3mNnk+N1pSUVKFJJOJgD3ufspwAHgY10cfyqhBV9mEFpjo97dTyc0z85nurpYeArxPcAsd59lZsWEVrK7yN3PANYAX444pcHdz3P3JcDj7n6mu59GaO6eG939RUITvf2ju0939+2tJ5pZHqGVz65x92mElhCIXFO6JnzNnwJf6Sp2kUhKFJJO3nT39eHPa4GyLo5f7u6H3b0aOMj7C1dtiOHcjnyE0DKVL5jZeuA64OSI/Q9HfD7VzFaZ2QZCK7md0kXdkwj9fFvC278GLojY/3j4eyw/t8hxtHCRpJOjEZ+bgXzgGO//wZQX5fiWiO0WevZ/x4A/uvsnO9lfF/F5EXClu78WXp1sZgx1R9MaezP6fy/dpBaFpLu3gA+HP388DvUfBgrCn18GzjWz8QDhcYeJnZxXAOw1s2yOXxs6sr5Im4Cy1rqBvwWe623wIqBEIfJvwOfM7EWgOA71LwR+b2bLw11Y1wOLzex1Qomjs4HlfwFeAf5IKAm0WgL8Y3jQelxrobs3EFqy97/D3VUtgNbflkDo8VgREYlKLQoREYlKg1oiPWRm9wHntiv+ibv/KhHxiMSLup5ERCQqdT2JiEhUShQiIhKVEoWIiESlRCEiIlEpUYiISFT/Bw9p+AeXJk6wAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(6, 4))\n",
    "ax = plt.axes()\n",
    "\n",
    "x = np.arange(0, 50)\n",
    "ax.loglog(np.arange(10000), GD_norm_lst, label='GD')\n",
    "ax.loglog(np.arange(10000), AGD_norm_lst, label='AGD')\n",
    "ax.legend()\n",
    "ax.set_xlabel('num_iteration')\n",
    "ax.set_ylabel('norm square')\n",
    "ax.set_title('loglog')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "abb5c435",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0004157828402733921"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(A @ x_GD) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "523a4bb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(A @ x_AGD) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "ee1e8c87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_A():\n",
    "    A = np.zeros((20, 20))\n",
    "    for i in range(20):\n",
    "        if i - 1 >= 0:\n",
    "            A[i][i - 1] = -1\n",
    "        if i + 1 <= 19:\n",
    "            A[i][i + 1] = -1\n",
    "        A[i][i] = 2\n",
    "    return A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "4c3e9392",
   "metadata": {},
   "outputs": [],
   "source": [
    "A = gen_A()\n",
    "x_2_AGD, AGD_norm_lst = AGD(np.ones(20,), A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "434c10ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(A @ x_2_AGD) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "a64cc6ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_2_GD, GD_norm_lst = GD(np.ones(20,), A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "fb27a4ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.023229089360321146"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(A @ x_2_GD) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "21b780a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'loglog scale')"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEbCAYAAADERMP2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAp50lEQVR4nO3deXydZZ3//9c7SZN0SZO2SUpXum9QWQxF9lIWUdm+qIg6IygDX5zB78h8HWQ2ZNT54vJ1HBn4qqiI/mYoMqBMcRkcaQtltQsVqG3pItC02CzQnW7J5/fHOQlpSE5O0nN60nPez8cjj5z7uu/7uj+5H20+ua7rvq9LEYGZmVl3inIdgJmZ9W9OFGZmlpIThZmZpeREYWZmKTlRmJlZSk4UZmaWkhOFFQRJr0g6PwP13Cvpy5mIKZskTZAUkkpyHYsd/ZwozMwsJScKMzNLyYnCCo6kMkn/ImlL8utfJJV12H+zpNeT+/4s2YUzpZu6rpO0XtIbkhZIGt1h34WS1kraLun/SXpc0p91U88cScsk7ZC0VdI/d9h3pqSnJW2TtEnSNcnyD0h6PnnOJkm3pfiZKyX9IPlzbZb0ZUnFvb97VoicKKwQ/R3wHuBE4ARgDvD3AJIuAv4KOB+YApzTXSWS5gG3A1cCo4BXgfuT+6qBB4G/AUYAa4HTU8T0LeBbETEUmAw8kKxnPPAr4F+BmmTMK5Pn7AY+AVQBHwA+Lenybur/EXAw+TOdBFwIdJm0zDpzorBC9HHgixHREBGNwD8Cf5rcdyXww4hYFRF7kvtS1XNPRKyIiH0kksJpkiYA7wdWRcRPI+IgcAfwxxR1HQCmSKqOiF0R8WyHa/wmIuZHxIGIaI6IlQARsTgiXoyI1oh4AZhPF4lN0kjgfcBnI2J3RDQA3wSu6ulGmYEThRWm0ST++m/zarKsbd+mDvs6fk5ZT0TsApqBMZ3ricTsm/Up6roWmAaskbRU0sXJ8nHAhq5OkHSqpEWSGiVtB24Aqrs49FhgAPB6svtqG/BdoDZFPGbtnCisEG0h8cuzzfhkGcDrwNgO+8alW4+kwSS6mTZ3rkeSOtV7iIhYFxEfJfHL+6vAg8n6NpHoiurKfcACYFxEVALfAdTFcZuAfUB1RFQlv4ZGxHEpfjazdk4UVojmA38vqSY5lnAr8G/JfQ8An5Q0U9Kg5L7u3Jc89sTkYPj/AZ6LiFeAXwCzJV2efJfhL4BjuqtI0p9IqomIVmBbsrgF+HfgfElXSiqRNELSicn9FcAbEbFX0hzgY13VHRGvA78GviFpqKQiSZMldTv+YtaRE4UVoi8Dy4AXgBeBFckyIuJXJMYTFgHrgWeS5+zrXElEPAb8A/AQiRbEZJL9/hHRBHwY+BqJ7qhZyWu+o56ki4BVknaRGNi+KiL2RsRrJMY7/jfwBomB7BOS5/w58EVJO0kktAdS/MyfAEqB3wNvkhhoH5XieLN28sJFZt2TNBN4CShLDkr3tZ4iEmMUH4+IRZmKz+xIcIvCrBNJ/0NSqaRhJMYLHulLkpD0XklVyW6pvyUxfvBsD6eZ9TtOFGbv9D+BRhJPG7UAn+5jPacl62gCLgEuj4i3MhKh2RHkriczM0vJLQozM0vJicLMzFLKq7nqJV0CXFJRUXHdtGnTch2OmdlRZfny5U0RUdO5PC/HKOrq6mLZsmW5DsPM7KgiaXlE1HUud9eTmZml5ERhZmYp5VWikHSJpLu3b9+e61DMzPJGXg1mR8QjwCN1dXXX5ToWMzv6HDhwgPr6evbu3ZvrULKqvLycsWPHMmDAgLSOz6tEYWZ2OOrr66moqGDChAkkZobPPxFBc3Mz9fX1TJw4Ma1z8qrryczscOzdu5cRI0bkbZIAkMSIESN61WpyojAz6yCfk0Sb3v6Medn1tGrLDmZ/4dG+nSwoKRLFya+SoqL2z4ltdbldUlRE0SHbesd2cVERxUW019nVMUUdjm0rLyspomxAceJ7SRFlJcWUJj+XD0hst5WXDSiitDgRi5kdnbZu3cpNN93Es88+y7BhwygtLeXmm29m2LBhXHbZZUyaNIk9e/YwcuRIbr75Zi6++OKeKz0MeZUo2t7MHjFuCh+uS7WCZfdaI2iN4GBr0Nqa+N7S2nG7tX27pcO+tw60dDqn9ZBjOp/T0qmuTL/3WFqcTCrJRDK4rJhBpSUMLitmcGkJg8sO/TyotDhZVsLgts+lJQwqK2Zo+QAqBw6gtMQNULNsiwguv/xyrr76au677z4AXn31VRYsWMCwYcM466yz+PnPfw7AypUrufzyyxk4cCDnnXde1mLKq0TR8amnWy+ZletweqW1NWiJdyaUgy2t7DvY9tWS+H4g8Xn/wU77Dhx63P7k57f2t7Jn/0F2729h976DNO/aw+79B9mzr4Xd+w+y90BrWjEOKi2mauAAhg4cQNWgRPKoGlhKZdvnQQOoHlJG9ZDS5PcyBpfl1T8xs6xbuHAhpaWl3HDDDe1lxx57LJ/5zGdYvHjxIceeeOKJ3Hrrrdx5551OFIWgqEgUIQYUH/lrH2xpZc+BRBLZvS/5ff/bn3fuPcC2PQfY/tYBtr2V+L59zwH+0LSb7W9tY9ueA+w72HWyGTigmOqKtxNH9ZAyaoaUMrKynNGVAxlVVc6oyoEMLS8piL5hO3r84yOr+P2WHRmtc9booXzhkuNSHrNq1SpOPvnktOs8+eST+frXv364oaXkRGGUFBcxtLiIoeXpPVPdlb0HWnhzz36ad+2ncdc+mnbuo2nXfpp27aNp1z6ad+1n0xt7eP61N2nevf8dXW2DS4sZVTWQUR0SyJiqgUyoHsyxwwdRU1HmRGIF6S/+4i948sknKS0t7TIhHIn5+pwoLCPKBxQzqnIgoyoH9njswZZWGnbu4/Xtb7Fl295Dvr++fS+rX99J0659h5wzcEAxx44YxPjhg5hQPTjxfcRgJtUMZlRluZOIZVxPf/lny3HHHcdDDz3Uvn3XXXfR1NREXd075uoD4Pnnn2fmzJlZjanfJwpJg4H/B+wHFkfEv+c4JDtMJcVFjK4ayOiqgbz72K6P2XewhS3b9vJq825ee2MPrzTt4bU3drOxaTeLX25kf4euriFlJUypHcLU2iFMHTmEqbUVTKkdwpiqgX76y4468+bN42//9m/59re/zac/nViFd8+ePV0e+8ILL/ClL32J73//+1mNKSeJQtI9wMVAQ0Qc36H8IuBbQDHw/Yj4CnAF8GBEPCLpJ4ATRQEoKylmYvVgJlYPfse+1tbgjzv28krzbjY07mb91p2sa9jF4pcb+Y/l9e3HDRxQzJTaIcwcVcGsUUM5bkwlM46poOIwutjMsk0SDz/8MDfddBNf+9rXqKmpYfDgwXz1q18FYMmSJZx00kns2bOH2tpa7rjjjqwOZEPuWhT3AncCP24rkFQM3AVcANQDSyUtAMYCLyYPazmyYVp/VFSk9hbJ6ZOrD9m3bc9+1jfsYl3DLtZt3cXLW3fy2OoGHlj2dgKZMGIQx42uZNboocwaPZTjR1dSU1F2pH8Ms26NGjWK+++/v8t9uZj0NCeJIiKekDShU/EcYH1EbASQdD9wGYmkMRZYid8ktx5UDSqlbsJw6iYMby+LCLbu2MfvX9/Oqs07+P3rO3hx83Z+8eLr7ceMqRrIieOqEl/jqzh+dCUDS3PwCJpZP9SfxijGAJs6bNcDpwJ3AHdK+gDwSHcnS7oeuB5g/PjxWQzTjjaSOKaynGMqy5k3Y2R7+Y69B1i9JZE0nt+0jZWvbWtPHsVFYsYxFZw4rooTxlVx8vgqJtcM8aC5FaT+lCi6+h8YEbEb+GRPJ0fE3ZJeBy4pLS19d8ajs7wztHwAp04awamTRrSXNe7cx8pN21i56U1WbtrGgpVb+PfnXgNg2KAB1E0YzpwJwzll4nCOGz2UAcVu5Fr+60+Joh7oOO/GWGBLbyrwehR2uGoqyrhg1kgumJVoebS2Bhsad7HitTdZ+sqbLH3lDf7791uBxGD5SeOrOGXCcE6ZMJyTxlf5TXTLS/3pX/VSYKqkicBm4CrgY72poG2upylTpmQhPCtERUVi6sgKpo6s4COnJLo0G3bsbU8av/3DG9yxcB0Rie6q40cP5T2TR3D65GpOmTCMQaX96b+YWd/oSLzV946LSvOBuUA1sBX4QkT8QNL7gX8h8XjsPRHxT32pv66uLpYtW5ahaM1S27H3ACtefTtxrNy0jQMtQUmROHFcFadPHsFpk6s5aXwV5bmYo8XStnr16qy/vNZfdPWzSloeEe94sy8niSJbOrQorlu3bl2uw7ECtWf/QZa98ibPbGzm6Q3NvFi/jdaAspIi6iYM47RJicTxrrGVHuPoZ/pTovjZz37GFVdcwerVq5kxYwYAv/3tb7n55pvZvHkzFRUVjBo1iq985SvMnj2b2267je9973vU1NSwe/duZs+ezZe//GVmzep6gtTeJIq8ahd7jML6g0GlJZw9rYazp9UAiRbHbze+0Z44/u+vXwZeZnBpMXMmDueMKdWcNbWGaSP9VJW9bf78+Zx55pncf//93HbbbWzdupUrr7yS++67j9NPPx2AJ598kg0bNjB79mwAbrrpJj73uc8B8JOf/IR58+bx4osvUlNTc1ix5FWi8BiF9UdDywdw/qyRnJ8cIH9j936e3djMMxuaeWpDE4t+sRpYTW1FGWdOreasqdWcOaXGLwEWsF27dvHUU0+xaNEiLr30Um677TbuvPNOrr766vYkAXDmmWd2W8dHPvIRfvGLX3Dffffxl3/5l4cVT14lCrco7GgwfHAp7589ivfPHgXAlm1v8eS6Jpasb2LRmgZ+umIzADNHDeWsZOI4ZcJwj28cab+6Bf74Ys/H9cYxs+F9X+nxsIcffpiLLrqIadOmMXz4cFasWMGqVau4+uqre3W5k08+mTVr1vQ12nZ5lSjcorCj0eiqgVx5yjiuPGUcra3Bqi07WLK+kSUvN3HvU69w9xMbKSspYs7E4cnEUcOMYyrcTZXH5s+fz2c/+1kArrrqKubPn/+OY0499VR27NjBhRdeyLe+9a0u68nUGHReJQq3KOxoV1QkZo+tZPbYSv587hT27D/Ic394gyUvN7FkXSP/55drgDVUDylLdlFVc/Y0d1NlRRp/+WdDc3MzCxcu5KWXXkISLS0tSOLqq69mxYoVXHbZZQA899xzPPjgg+3Lonbl+eef73Z68t7Iq0Rhlm8GlZZw7vRazp1eC8Aft+9lybpGlqxr4omXG/nZ84luquPHDOWcaTXMnV7LSeOqKPHTVEetBx98kE984hN897vfbS8755xzuPDCC/mTP/kT3vve97aPU3Q3/TjAQw89xK9//Wu+8Y1vHHZMeZUo3PVk+e6YynI+XDeOD9cluql+//oOHn+5kcVrG/jO4xu5a9EGKspLOGtqNXOn1XLO9BpGDi3PddjWC/Pnz+eWW245pOyDH/wg9913Hz/5yU/4/Oc/z+bNm6mtraW6uppbb721/bhvfvOb/Nu//Ru7d+/m+OOPZ+HChYf9xBPk2XsUbfzCnRWi7W8d4Kn1TTy+tpHFLzewdUdilcAZx1Qwd3otc6fX8O5jh/ndjRT603sU2Vaw71GYFbLKgQPan6aKCNb8cWd7a+P7Szbyncc3MKSshDOmjGDu9FrOmVbD6Kqel641c6Iwy0OSmDlqKDNHDeWGcyazc+8Bnt7QzOK1jTy+toFHVyUmNpw2ckh70qibMIyyEj+Ca++UV4nCYxRmXasoH8B7jzuG9x53DBHB+oZdLE52UbU9gjuotJjTJ1czd3oN586oZYxbG5aUV4nCj8ea9Ux6e0bc686exO59B3lmQzOLX25g8dpGfrM60dqYcUwF586oZd6MwnqSKiLy/h2V3o5N51WiMLPeG1xW0j7FSERi/Y2FaxpYuKaB7z2xkW8v3kDlwAGcM62GeTMS3VTDBpfmOuysKC8vp7m5mREjRuRtsogImpubKS9P/2k4P/VkZt3asfcAS15uYuGaBh5/uYGmXfspEpw0fhjzZiTe75g5Kn/eEj9w4AD19fXs3bs316FkVXl5OWPHjmXAgAGHlBfENONtnCjMMq+1NXhh83YWrmlg0ZoGXty8HYBRleXMnZ7oojpjyggv1nQUc6Iws4xq2LGXxWsbWbimgSXrGtm9v4XSkiLeM2kE86bXMG/GSMaPGJTrMK0XCiJReOEis9zYf7CVpa+80d7a2Ni0G4DJNYMTXVQzajllwnC/7NfPFUSiaOMWhVluvdK0O5E01jbw3MY32N/SSkVZCWdNq+bc6bXMnV7riQz7IScKM8uJ3fsO8mRyrY1FaxNTi0jwrjGVzJsxkvNm1nLc6KF5MyB+NHOiMLOci0ist7FoTQML1zawctM2IhID4vNm1HL+zJGcNnmEF2nKEScKM+t3mnbtY9GaBh5b3cAT6xrZs7+FgQOKOXNqNefNqGXezFpqKzz77ZHiRGFm/dq+gy08u/ENHlu9lcdWN7B521sAnDC2kvNmJrqoZo1yF1U2HbWJQtIk4O+Ayoj4UDrnOFGYHd3aZr99bPVWfrO6gd/VJ7qoRleWM29mLefNHMlpk9xFlWk5SRSS7gEuBhoi4vgO5RcB3wKKge9HRI9rDkp60InCrDA17kx0Uf1m9VaWrGvirQMtDCot5swp1Zw/cyTnzvBTVJmQq0RxNrAL+HFbopBUDLwMXADUA0uBj5JIGrd3quJTEdGQPM+JwszYe6CFZzY2t3dRvb59LxKcMLaK85OtjRnH5M+0IkdSzrqeJE0Aft4hUZwG3BYR701u/w1ARHROEp3rSZkoJF0PXA8wfvz4d7/66quZ+QHMrN+KSCwH+9jqBh5bvZXf1SemFRlTNZDzkknjPZOGe52NNPWnRPEh4KKI+LPk9p8Cp0bEjd2cPwL4JxItkO/3lFDALQqzQtWwYy8L1zTwm9UNPLm+kb0HWhlcWsxZU2s4b2biDfHqIe6i6k5/Wgq1q/Zgt9kqIpqBG9Kq2AsXmRW02qHlXDVnPFfNGc/eAy08vaGJ36xuYOHqBv5r1R+R4MRxVZw/cyTzZtS6iypNuUgU9cC4DttjgS05iMPM8lj5gGLmzRjJvBkjicsTL/o9trqBhWu28vVH1/L1R9cypmog82bUct7MWt7jp6i6lYuupxISg9nnAZtJDGZ/LCJWZeqa7noys1Qaduxl0dpkF1Wnp6jauqgK8UW/XD31NB+YC1QDW4EvRMQPJL0f+BcSTzrdExH/lKHrefZYM+uVjk9RLVzdwJbtiUWLThhXxXnJ1kahvOh31L5w1xduUZhZX0QEq1/fycI1h77o1zYX1XkzazltUjUDS/Ozi6ogEoVbFGaWSY0797FobeLR2yXrmtizv4WykiJOmzyifSnYccPzZ3GmgkgUbdyiMLNM23ewhec2vsGitYnFmV5p3gN0WJxpei11E4ZTWnL0Ls7kRGFmlkF/aNrdvsZG2+JMQ8pKOHNKNefOqGHu9FpGDj26BsQLIlG468nMcmH3voM8vaGZhWsaWLw2Ma0IwHGjh3Lu9MRTVCeOq6K4qH8PiBdEomjjFoWZ5UpEsHbrzkTSWNPI8tfepKU1qBo0gHOm1TBvRi1nT61h2ODSXIf6Dk4UZmY5sH3PAZ5Y18iitQ08vraR5t37KUq+IT5vRmL98P6yFGxBJAp3PZlZf9baGryweXv72MYLyUkMayvKkl1UNZwxpZqK8gE5ia8gEkUbtyjM7GjQuHMfi9c2sHhtI0+83MjOfQcZUCxOmTCcc6fX8v53jWJM1cAjFo8ThZlZP3agpZXlr77Z/vjty1t3ATBnwnAuO2k0H5g9iqpB2R3XcKIwMzuKvNq8mwUrt/Dwys1saNzNgGJxzrRaLjtxNBfMGpmVCQwLIlF4jMLM8k1EYubbh5/fzILfbaFh5z6qBg3gf5w0ho/OGc+0kRUZu1ZBJIo2blGYWT5qaQ2e3tDE/Us38etVf+RAS3Dy+CqumjOei981ikGlh7dyhBOFmVkead61j5+u2Mz8pa+xsXE3FWUlXHHyGK47exJjh/Vt/qnuEsXROymJmVkBGzGkjOvOnsRjf3UOD/zP0zhvZi3zf7uJbXsOZPxauVjhzszMMkQScyYOZ87E4dx26f6sPBmVVy0KSZdIunv79u25DsXM7IjL1uOzeZUoIuKRiLi+srIy16GYmeWNvEoUZmaWeU4UZmaWkhOFmZml5ERhZmYpHRWJQtLlkr4n6T8lXZjreMzMCkmPiULSIEn/IOl7ye2pki5O9wKS7pHUIOmlTuUXSVorab2kW1LVEREPR8R1wDXAR9K9tpmZHb50WhQ/BPYBpyW364Ev9+Ia9wIXdSyQVAzcBbwPmAV8VNIsSbMl/bzTV22HU/8+eZ6ZmR0h6byZPTkiPiLpowAR8ZZ6sWZfRDwhaUKn4jnA+ojYCCDpfuCyiLgdeEdrJXm9rwC/iogV6V7bzMwOXzotiv2SBgIBIGkyiRbG4RgDbOqwXZ8s685ngPOBD0m6oasDJF0vaZmkZY2NjYcZnpmZtUmnRfEF4L+AcZL+HTiDxFjB4eiqRdLtNLYRcQdwR6oKI+JuSa8Dl5SWlr77MOMzM7OklC0KSUXAMOAKEslhPlAXEYsP87r1wLgO22OBLYdZp6fwMDPLgpSJIiJagRsjojkifhERP4+IpgxcdykwVdJESaXAVcCCw63UkwKamWVeOmMU/y3pc5LGSRre9pXuBSTNB54Bpkuql3RtRBwEbgQeBVYDD0TEqj79BB24RWFmlnk9rnAn6Q9dFEdETMpOSH3nNbPNzPrOS6GamVlK3SWKtFa4k3Q8iRfjytvKIuLHmQsvMzq0KHIdiplZ3khnCo8vAP+a/DoX+BpwaZbj6hOPUZiZZV46g9kfAs4D/hgRnwROAMqyGlUf+aknM7PMSydRvJV8TPagpKFAA9DvBrLBLQozs2xIZ4ximaQq4HvAcmAX8NtsBmVmZv1Hj4kiIv48+fE7kv4LGBoRL2Q3rL7xYLaZWealM5h9dtsXMB6oSn7ud9z1ZGaWeel0Pf11h8/lJKYIXw7My0pEZmbWr6TT9XRJx21J40g8ImtmZgWgL2tm1wPHZzqQTPDjsWZmmddji0LSv/L2WhFFwInA77IYU59FxCPAI3V1ddflOhYzs3yR1uOxHT4fBOZHxFNZisfMzPqZdMYofnQkAjEzs/4pna6nF+l6mVKRmG78XRmPyszM+o10up5+lfz+/yW/fxzYA7ilYWZWANJJFGdExBkdtm+R9FREfDFbQfWV38w2M8u8dB6PHSzpzLYNSacDg7MXUt/5zWwzs8xLp0VxLXCPpLbfvtuAT2UtIjMz61fSeeppOXBCcopxRYTfZjMzKyDpTAr4l8kksRP4hqQVki7MfmhmZtYfpDNG8amI2AFcCNQCnwS+ktWozMys30gnUSj5/f3ADyPidx3Ksk7STEnfkfSgpE8fqeuamVlCOoliuaRfk0gUj0qqAFrTqVzSPZIaJL3UqfwiSWslrZd0S6o6ImJ1RNwAXAnUpXNdMzPLnHSfejoR2BgReySNINH9lI57gTuBH7cVSCoG7gIuIDET7VJJC4Bi4PZO538qIhokXQrckqzLzMyOoHSeemoFVnTYbgaa06k8Ip6QNKFT8RxgfURsBJB0P3BZRNwOXNxNPQuABZJ+AdzX1TGSrgeuBxg/fnw64ZmZWRrSaVFk2hhgU4fteuDU7g6WNBe4AigDftndcRFxN3A3QF1dXVdzU5mZWR/kIlF0NRDe7S/2iFgMLE6rYk/hYWaWcWklCknDgHEdj4+IFd2fkVJ9sq42Y4EtfazLzMyyLJ1pxr8EXANs4O2//AOY18drLgWmSpoIbAauAj7Wx7oO4RXuzMwyL50WxZXA5IjY39vKJc0H5gLVkuqBL0TEDyTdCDxK4kmneyJiVW/r7uZ67noyM8swRaQe95X0EPDpiGg4MiEdvrq6uli2bFnPB5qZWTtJyyPiHe+rpdOiuB14PvnS3L62woi4NIPxZYRbFGZmmZdOovgR8FXgRdJ8IztXPEZhZpZ56SSKpoi4I+uRmJlZv5ROolgu6XZgAYd2PfX18discdeTmVnmpTOYvaiL4oiIvj4em3UezDYz670+DWYnJ/BbEBHfzFpkZmbWr6WcZjwiWoB+93RTdyRdIunu7du9WquZWaaksx7F05LulHSWpJPbvrIeWR9ExCMRcX1lZWWuQzEzyxvpDGafnvz+xQ5lhzOFh5mZHUXSWY/i3CMRSCb4qSczs8zrsetJUqWkf5a0LPn1DUn9sm/HXU9mZpmXzhjFPcBOEpMDXgnsAH6YzaDMzKz/SGeMYnJEfLDD9j9KWpmleMzMrJ9Jp0XxlqQz2zYknQG8lb2QzMysP0mnRXED8OPkuISAN0gsZGRmZgUgnaeefgecIGlocntH1qPqIz/1ZGaWeekshVoGfBCYAJRIAiAivpjitJzwNONmZpmXTtfTfwLbgeV0mD3WzMwKQzqJYmxEXJT1SMzMrF9Kd66n2VmPxMzM+qV0WhRnAtdI+gOJrieRWI/iXVmNzMzM+oV0EsX7sh5FDyQNBp4AvhARP891PGZmhaTHrqeIeLWrr3Qql3SPpAZJL3Uqv0jSWknrJd2SRlWfBx5I55pmZpZZ6bQoDse9wJ3Aj9sKkqvm3QVcANQDSyUtAIqB2zud/yngXcDvgfIsx2pmZl3IaqKIiCckTehUPAdYHxEbASTdD1wWEbcDF3euQ9K5wGBgFonpRH4ZEa1dHHc9cD3A+PHjM/pzmJkVsmy3KLoyBtjUYbseOLW7gyPi7wAkXQM0dZUkksfdDdwNUFdXF5kK1sys0OUiUaiLsh5/sUfEvT1W7Ck8zMwyLp33KDKtHhjXYXsssCUTFXvhIjOzzMtFolgKTJU0UVIpcBWwIBMVS7pE0t3bt2/PRHVmZkaWE4Wk+cAzwHRJ9ZKujYiDwI3Ao8Bq4IGIWJXNOMzMrO8UkX/jvnV1dbFs2bJch2FmdlSRtDwi6jqX56LrKWvc9WRmlnl5lSg8mG1mlnl5lSjMzCzz8ipRuOvJzCzz8ipRuOvJzCzz8ipRmJlZ5uVVonDXk5lZ5uVVonDXk5lZ5uVVojAzs8xzojAzs5TyKlF4jMLMLPPyKlF4jMLMLPPyKlGYmVnmOVGYmVlKThRmZpZSXiUKD2abmWVeXiUKD2abmWVeXiUKMzPLPCcKMzNLyYnCzMxScqIwM7OU+n2ikDRX0hJJ35E0N9fxmJkVmqwmCkn3SGqQ9FKn8oskrZW0XtItPVQTwC6gHKjPVqxmZta1kizXfy9wJ/DjtgJJxcBdwAUkfvEvlbQAKAZu73T+p4AlEfG4pJHAPwMfz3LMZmbWQVYTRUQ8IWlCp+I5wPqI2Agg6X7gsoi4Hbg4RXVvAmVZCdTMzLqV7RZFV8YAmzps1wOndnewpCuA9wJVJFon3R13PXA9wPjx4zMRp5mZkZtEoS7KoruDI+KnwE97qjQi7pb0OnBJaWnpuw8jPjMz6yAXTz3VA+M6bI8FtmSiYk/hYWaWeblIFEuBqZImSioFrgIWZKJiTwpoZpZ52X48dj7wDDBdUr2kayPiIHAj8CiwGnggIlZl4npuUZiZZV62n3r6aDflvwR+menrSboEuGTKlCmZrtrMrGD1+zeze8MtCjOzzMurROExCjOzzMurROEWhZlZ5uVVonCLwsws8/IqUbhFYWaWeXmVKMzMLPPyKlG468nMLPPyKlG468nMLPPyKlGYmVnmOVGYmVlKeZUoPEZhZpZ5eZUoPEZhZpZ5eZUozMws85wozMwsJScKMzNLyYnCzMxSyqtE4aeezMwyL68ShZ96MjPLvLxKFGZmlnlOFGZmlpIThZmZpeREYWZmKZXkOoCeSCoCvgQMBZZFxI9yHJKZWUHJaotC0j2SGiS91Kn8IklrJa2XdEsP1VwGjAEOAPXZitXMzLqW7RbFvcCdwI/bCiQVA3cBF5D4xb9U0gKgGLi90/mfAqYDz0TEdyU9CDzW41W3vQo/u6Htim0X7rTd8YTujlGOjqHrY8zMelL3KRg+MaNVZjVRRMQTkiZ0Kp4DrI+IjQCS7gcui4jbgYs71yGpHtif3Gzp7lqSrgeuBzhxdCm8+hREeyRtAR263VVZVo/pIZ7ocGyXZWZmPZj+/qMrUXRjDLCpw3Y9cGqK438K/Kuks4AnujsoIu4G7gaoq6sLPrssA6GamVkuEkVX/Sjd/tkcEXuAa9OqWLoEuGTKlCl9DM3MzDrLxeOx9cC4DttjgS05iMPMzNKQi0SxFJgqaaKkUuAqYEEmKvZcT2ZmmZftx2PnA88A0yXVS7o2Ig4CNwKPAquBByJiVYau59ljzcwyTJGHT9XU1dXFsmUezDYz6w1JyyOirnN5Xk3h4RaFmVnm5VWi8BiFmVnm5VWiMDOzzOv3kwL2Rtt7FMAOSeuSxZVA576ozmUdt6uBpiyF2FUsmTon1XHd7Uvn3nRV1p/vV7rnZep+dVXu+5V6X6Hfr1T7c32/ju2yNCLy+gu4u6eyjtskZqg9YrFk6pxUx3W3L517c7Tdr3TPy9T96un+FPL96m5fod+vVPv76/0qhK6nR9Io6+qYbOjLddI9J9Vx3e1L5950Vdaf71e652XqfnVV7vuVel+h369U+/vl/crLx2MPh6Rl0cXjYdY136/e8f3qHd+v3snW/SqEFkVv3Z3rAI4yvl+94/vVO75fvZOV++UWhZmZpeQWhZmZpeREYWZmKTlRmJlZSk4UPZA0WNKPJH1P0sdzHU9/J2mSpB8k1ze3Hki6PPlv6z8lXZjrePo7STMlfUfSg5I+net4jgbJ32HLJb1jqel0FWSikHSPpAZJL3Uqv0jSWknrJd2SLL4CeDAirgMuPeLB9gO9uV8RsTEi0lqRMF/18n49nPy3dQ3wkRyEm3O9vF+rI+IG4EqgIB+b7eXvL4DPAw8czjULMlEA9wIXdSyQVAzcBbwPmAV8VNIsEivwta3x3XIEY+xP7iX9+2V9u19/n9xfiO6lF/dL0qXAk8BjRzbMfuNe0rxfks4Hfg9sPZwLFmSiiIgngDc6Fc8B1if/It4P3A9cRmLp1rHJY3y/3tbd/Sp4vblfSvgq8KuIWHGkY+0PevvvKyIWRMTpQEF2Bffyfp0LvAf4GHCdpD79DsurSQEP0xjebjlAIkGcCtwB3CnpAxy5qQWOBl3eL0kjgH8CTpL0NxFxe06i63+6+/f1GeB8oFLSlIj4Ti6C64e6+/c1l0R3cBnwyyMfVr/V5f2KiBsBJF0DNEVEa18qd6J4m7ooi4jYDXzySAdzFOjufjUDNxzpYI4C3d2vO0j8MWKH6u5+LQYWH9lQjgpd3q/2DxH3Hk7lBdmV0o16YFyH7bHAlhzFcjTw/eod36/e8f3qnazeLyeKty0FpkqaKKkUuApYkOOY+jPfr97x/eod36/eyer9KshEIWk+8AwwXVK9pGsj4iBwI/AosBp4ICJW5TLO/sL3q3d8v3rH96t3cnG/PCmgmZmlVJAtCjMzS58ThZmZpeREYWZmKTlRmJlZSk4UZmaWkhOFmZml5ERhZmYpOVGYZZikLyand0bSZyUNymDdl3ecnrzjtcyyxS/cmWWRpFeAuoho6sU5xRHR5donku4Ffh4RXkHQjhi3KKwgSJogaXVy2dFVkn4taaCkxZLqksdUJ3+xI+kaSQ9LekTSHyTdKOmvJD0v6VlJw1Nc615JH5L0v4DRwCJJi5L7LpT0jKQVkv5D0pBk+SuSbpX0JPBhSddJWirpd5IekjRI0ukkVln8uqSVkia3XStZx3nJ+F5MroJW1qHuf0xe80VJM7J3py0fOVFYIZkK3BURxwHbgA/2cPzxJBZ8mUNijY09EXESiXl2PtHTxZJTiG8Bzo2IcyVVk1jJ7vyIOBlYBvxVh1P2RsSZEXE/8NOIOCUiTiAxd8+1EfE0iYne/joiToyIDW0nSionsfLZRyJiNoklBDquKd2UvOa3gc/1FLtZR04UVkj+EBErk5+XAxN6OH5RROyMiEZgO28vXPViGud25T0klql8StJK4Grg2A77f9Lh8/GSlkh6kcRKbsf1UPd0Ej/fy8ntHwFnd9j/0+T3dH5us0N44SIrJPs6fG4BBgIHefsPpvIUx7d22G6lb/93BPx3RHy0m/27O3y+F7g8In6XXJ1sbhp1p9IWewv+f2+95BaFFbpXgHcnP38oC/XvBCqSn58FzpA0BSA57jCtm/MqgNclDeDQtaE71tfRGmBCW93AnwKPH27wZuBEYfZ/gU9LehqozkL9dwO/krQo2YV1DTBf0gskEkd3A8v/ADwH/DeJJNDmfuCvk4PWk9sKI2IviSV7/yPZXdUKeP1tywg/HmtmZim5RWFmZil5UMusjyTdBZzRqfhbEfHDXMRjli3uejIzs5Tc9WRmZik5UZiZWUpOFGZmlpIThZmZpeREYWZmKf3/T2CUG7Qek8EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(6, 4))\n",
    "ax = plt.axes()\n",
    "\n",
    "x = np.arange(0, 50)\n",
    "ax.loglog(np.arange(10000), GD_norm_lst, label='GD')\n",
    "ax.loglog(np.arange(10000), AGD_norm_lst, label='AGD')\n",
    "ax.legend()\n",
    "ax.set_xlabel('num_iteration')\n",
    "ax.set_ylabel('norm square')\n",
    "ax.set_title('loglog scale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe5bd11e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}