From b398241434decaa5912162096e5976e800f74f4a Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 16:39:12 -0400
Subject: [PATCH 001/170] exchange .ipynb with .py and add a demo document

---
 docs/demos/pinn_forward/heat.rst              | 110 +++++
 examples/pinn_forward/Heat.py                 | 121 ++++++
 .../heat_conduction_1d_uniform_bar.ipynb      | 397 ------------------
 3 files changed, 231 insertions(+), 397 deletions(-)
 create mode 100644 docs/demos/pinn_forward/heat.rst
 create mode 100644 examples/pinn_forward/Heat.py
 delete mode 100644 examples/pinn_forward/heat_conduction_1d_uniform_bar.ipynb

diff --git a/docs/demos/pinn_forward/heat.rst b/docs/demos/pinn_forward/heat.rst
new file mode 100644
index 000000000..f1b248e91
--- /dev/null
+++ b/docs/demos/pinn_forward/heat.rst
@@ -0,0 +1,110 @@
+Heat equation in 1D with Dirichilet boundary conditions and periodic(sinusoidal) initial conditions
+======================================================================================================
+
+Problem setup
+--------------
+
+We will solve a Heat equation:
+
+.. math:: \alpha \frac{\partial^2u}{\partial x^2}-\frac{\partial u}{\partial t} = 0, \qquad x \in [-1, 1], \quad t \in [0, 1]
+
+where :math:`\alpha` is the thermal diffusivity constant.
+
+With Dirichlet boundary conditions:
+
+.. math:: u(0,t) = u(1,t)=0,
+
+and periodic(sinusoidal) inital condition:
+
+.. math:: u(x,0) = \sin (\frac{n\pi x}{L}),\qquad 0<x<L, \quad n = 1,2,.....
+
+where :math:`L` is the length of the bar, :math:`n` is the frequency of the sinusoidal initial conditions.
+
+The exact solution is :math:`u(x,t) = \exp{\frac{-n ^2\pi ^2 \alpha t}{L^2}}\sin (\frac{n\pi x}{L}))`.
+
+Implementation
+-----------------
+This description goes through the implementation of a solver for the above described Heat equation step-by-step.
+
+First, the DeepXDE and TensorFlow (``tf``) modules are imported:
+
+.. code-block:: python
+
+    import deepxde as dde
+    from deepxde.backend import tf
+
+We begin by defining a computational geometry and time domain. We can use a built-in class ``Interval`` and ``TimeDomain`` and we combine both the domains using ``GeometryXTime`` as follows
+
+.. code-block:: python
+
+    geom = dde.geometry.Interval(0, L)
+    timedomain = dde.geometry.TimeDomain(0, 1)
+    geomtime = dde.geometry.GeometryXTime(geom, timedomain)
+
+Next, we define the parameters of the equation:
+
+.. code-block:: python
+
+    a = 0.4
+    L = 1
+
+Next, we express the PDE residual of the Heat equation:
+
+.. code-block:: python
+
+    def pde(x, y):
+        dy_t = dde.grad.jacobian(y, x, i=0, j=1)
+        dy_xx = dde.grad.hessian(y, x, i=0, j=0)
+        return dy_t - a*dy_xx
+
+The first argument to ``pde`` is 2-dimensional vector where the first component(``x[:,0]``) is :math:`x`-coordinate and the second componenet (``x[:,1]``) is the :math:`t`-coordinate. The second argument is the network output, i.e., the solution :math:`u(x,t)`, but here we use ``y`` as the name of the variable.
+
+Next, we consider the boundary/initial condition. ``on_boundary`` is chosen here to use the whole boundary of the computational domain in considered as the boundary condition. We include the ``geomtime`` space, time geometry created above and ``on_boundary`` as the BCs in the ``DirichletBC`` function of DeepXDE. We also define ``IC`` which is the inital condition for the burgers equation and we use the computational domain, initial function, and ``on_initial`` to specify the IC. 
+
+.. code-block:: python
+
+    bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)
+    ic = dde.icbc.IC(geomtime, lambda x: np.sin(n*np.pi*x[:, 0:1]/L), lambda _, on_initial: on_initial)
+
+Now, we have specified the geometry, PDE residual, and boundary/initial condition. We then define the ``TimePDE`` problem as
+
+.. code-block:: python
+
+     data = dde.data.TimePDE(
+        geomtime, pde, [bc, ic], num_domain=2540, num_boundary=80, num_initial=160, num_test=2540
+    )
+
+The number 2540 is the number of training residual points sampled inside the domain, and the number 80 is the number of training points sampled on the boundary. We also include 160 initial residual points for the initial conditions.
+
+Next, we choose the network. Here, we use a fully connected neural network of depth 4 (i.e., 3 hidden layers) and width 20:
+
+.. code-block:: python
+
+    net = dde.nn.FNN([2] + [20] * 3 + [1], "tanh", "Glorot normal")
+
+Now, we have the PDE problem and the network. We build a ``Model`` and choose the optimizer and learning rate:
+
+.. code-block:: python
+
+    model = dde.Model(data, net)
+    model.compile("adam", lr=1e-3)
+    
+   
+We then train the model for 20000 iterations:
+
+.. code-block:: python
+
+    losshistory, train_state = model.train(epochs=20000)
+    
+After we train the network using Adam, we continue to train the network using L-BFGS to achieve a smaller loss:
+
+.. code-block:: python
+
+    model.compile("L-BFGS-B")
+    losshistory, train_state = model.train() 
+
+Complete code
+--------------
+
+.. literalinclude:: ../../../examples/pinn_forward/Heat.py
+  :language: python
diff --git a/examples/pinn_forward/Heat.py b/examples/pinn_forward/Heat.py
new file mode 100644
index 000000000..a146693ea
--- /dev/null
+++ b/examples/pinn_forward/Heat.py
@@ -0,0 +1,121 @@
+# Import libraries:
+import matplotlib.pyplot as plt
+import numpy as np
+
+import deepxde as dde
+##from deepxde.backend import tf
+
+def heat_eq_exact_solution(x, t):
+    """
+    Returns the exact solution for a given x and t (for sinusoidal initial conditions).
+    
+    Parameters
+    ----------
+    x : np.ndarray
+    t : np.ndarray
+    """
+    
+    return np.exp(-(n**2*np.pi**2*a*t)/(L**2))*np.sin(n*np.pi*x/L)
+
+def gen_exact_solution():
+    """
+    Generates exact solution for the heat equation for the given values of x and t.
+    """
+    
+    # Number of points in each dimension:
+    x_dim, t_dim = (256, 201)
+
+    # Bounds of 'x' and 't':
+    x_min, t_min = (0, 0.)
+    x_max, t_max = (L, 1.)
+
+    # Create tensors:
+    t = np.linspace(t_min, t_max, num=t_dim).reshape(t_dim, 1)
+    x = np.linspace(x_min, x_max, num=x_dim).reshape(x_dim, 1)
+    usol = np.zeros((x_dim, t_dim)).reshape(x_dim, t_dim)
+
+    # Obtain the value of the exact solution for each generated point:
+    for i in range(x_dim):
+        for j in range(t_dim):
+            usol[i][j] = heat_eq_exact_solution(x[i],t[j])
+
+    # Save solution:
+    np.savez('heat_eq_data', x=x, t=t, usol=usol)
+    # Load solution:
+    data = np.load('heat_eq_data.npz')
+
+
+def gen_testdata():
+    """
+    Import and preprocess the dataset with the exact solution.
+    """
+    
+    # Load the data:
+    data = np.load('heat_eq_data.npz')
+    
+    # Obtain the values for t, x, and the excat solution:
+    t, x, exact = data["t"], data["x"], data["usol"].T
+    
+    # Process the data and flatten it out (like labels and features):
+    xx, tt = np.meshgrid(x, t)
+    X = np.vstack((np.ravel(xx), np.ravel(tt))).T
+    y = exact.flatten()[:, None]
+    
+    return X, y
+
+def main():
+
+    def pde(x, y):
+        """
+        Expresses the PDE residual of the heat equation.        
+        """
+        dy_t = dde.grad.jacobian(y, x, i=0, j=1)
+        dy_xx = dde.grad.hessian(y, x, i=0, j=0)
+        return dy_t - a*dy_xx
+
+    # Computational geometry:
+    geom = dde.geometry.Interval(0, L)
+    timedomain = dde.geometry.TimeDomain(0, 1)
+    geomtime = dde.geometry.GeometryXTime(geom, timedomain)
+
+    # Initial and boundary conditions:
+    bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)
+    ic = dde.icbc.IC(
+        geomtime, lambda x: np.sin(n*np.pi*x[:, 0:1]/L), lambda _, on_initial: on_initial
+        )
+
+    # Define the PDE problem and configurations of the network:
+    data = dde.data.TimePDE(
+        geomtime, pde, [bc, ic], num_domain=2540, num_boundary=80, num_initial=160, num_test=2540
+    )
+    net = dde.nn.FNN([2] + [20] * 3 + [1], "tanh", "Glorot normal")
+    model = dde.Model(data, net)
+
+    # Build and train the model:
+    model.compile("adam", lr=1e-3)
+    model.train(epochs=20000)
+    model.compile("L-BFGS")
+    losshistory, train_state = model.train()
+
+    # Plot/print the results
+    dde.saveplot(losshistory, train_state, issave=True, isplot=True)
+    X, y_true = gen_testdata()
+    y_pred = model.predict(X)
+    f = model.predict(X, operator=pde)
+    print("Mean residual:", np.mean(np.absolute(f)))
+    print("L2 relative error:", dde.metrics.l2_relative_error(y_true, y_pred))
+    np.savetxt("test.dat", np.hstack((X, y_true, y_pred)))
+
+if __name__ == "__main__":
+
+    # Problem parameters:
+    a = 0.4 # Thermal diffusivity
+    L = 1 # Length of the bar
+    n = 1 # Frequency of the sinusoidal initial conditions
+
+    # Generate a dataset with the exact solution (if you dont have one):
+    gen_exact_solution()
+
+    # Solve the equation:
+    main()
+    
\ No newline at end of file
diff --git a/examples/pinn_forward/heat_conduction_1d_uniform_bar.ipynb b/examples/pinn_forward/heat_conduction_1d_uniform_bar.ipynb
deleted file mode 100644
index b2826e6e6..000000000
--- a/examples/pinn_forward/heat_conduction_1d_uniform_bar.ipynb
+++ /dev/null
@@ -1,397 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "Z2PweKqXHaGp",
-    "outputId": "85333610-0880-4826-d3da-e56ac4bcaa40"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: deepxde in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (0.13.6)\n",
-      "Requirement already satisfied: scipy in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from deepxde) (1.5.4)\n",
-      "Requirement already satisfied: matplotlib in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from deepxde) (3.4.3)\n",
-      "Requirement already satisfied: scikit-optimize in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from deepxde) (0.9.0)\n",
-      "Requirement already satisfied: scikit-learn in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from deepxde) (0.24.1)\n",
-      "Requirement already satisfied: numpy in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from deepxde) (1.19.4)\n",
-      "Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from matplotlib->deepxde) (2.4.7)\n",
-      "Requirement already satisfied: cycler>=0.10 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from matplotlib->deepxde) (0.10.0)\n",
-      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from matplotlib->deepxde) (8.2.0)\n",
-      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from matplotlib->deepxde) (2.8.2)\n",
-      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from matplotlib->deepxde) (1.3.1)\n",
-      "Requirement already satisfied: joblib>=0.11 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from scikit-learn->deepxde) (1.0.0)\n",
-      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from scikit-learn->deepxde) (2.1.0)\n",
-      "Requirement already satisfied: pyaml>=16.9 in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from scikit-optimize->deepxde) (21.10.1)\n",
-      "Requirement already satisfied: six in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from cycler>=0.10->matplotlib->deepxde) (1.16.0)\n",
-      "Requirement already satisfied: PyYAML in c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages (from pyaml>=16.9->scikit-optimize->deepxde) (5.4.1)\n",
-      "Note: you may need to restart the kernel to use updated packages.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -illow (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -illow (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -illow (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -illow (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n",
-      "WARNING: Ignoring invalid distribution -illow (c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages)\n"
-     ]
-    }
-   ],
-   "source": [
-    "pip install deepxde"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "id": "Eq4raiEN-hu8"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Using backend: tensorflow\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Import libraries:\n",
-    "import matplotlib.pyplot as plt\n",
-    "import numpy as np\n",
-    "\n",
-    "import deepxde as dde\n",
-    "from deepxde.backend import tf\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "id": "fDLgsURPSEg6"
-   },
-   "outputs": [],
-   "source": [
-    "def heat_eq_exact_solution(x, t):\n",
-    "    \"\"\"\n",
-    "    Returns the exact solution for a given x and t (for sinusoidal initial conditions).\n",
-    "    \n",
-    "    Parameters\n",
-    "    ----------\n",
-    "    x : np.ndarray\n",
-    "    t : np.ndarray\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    return np.exp(-(n**2*np.pi**2*a*t)/(L**2))*np.sin(n*np.pi*x/L)\n",
-    "\n",
-    "def gen_exact_solution():\n",
-    "    \"\"\"\n",
-    "    Generates exact solution for the heat equation for the given values of x and t.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    # Number of points in each dimension:\n",
-    "    x_dim, t_dim = (256, 201)\n",
-    "\n",
-    "    # Bounds of 'x' and 't':\n",
-    "    x_min, t_min = (0, 0.)\n",
-    "    x_max, t_max = (L, 1.)\n",
-    "\n",
-    "    # Create tensors:\n",
-    "    t = np.linspace(t_min, t_max, num=t_dim).reshape(t_dim, 1)\n",
-    "    x = np.linspace(x_min, x_max, num=x_dim).reshape(x_dim, 1)\n",
-    "    usol = np.zeros((x_dim, t_dim)).reshape(x_dim, t_dim)\n",
-    "\n",
-    "    # Obtain the value of the exact solution for each generated point:\n",
-    "    for i in range(x_dim):\n",
-    "        for j in range(t_dim):\n",
-    "            usol[i][j] = heat_eq_exact_solution(x[i],t[j])\n",
-    "\n",
-    "    # Save solution:\n",
-    "    np.savez('heat_eq_data', x=x, t=t, usol=usol)\n",
-    "    data = np.load('heat_eq_data.npz')\n",
-    "\n",
-    "\n",
-    "def gen_testdata():\n",
-    "    \"\"\"\n",
-    "    Import and preprocess the dataset with the exact solution.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    # Load the data:\n",
-    "    data = np.load('heat_eq_data.npz')\n",
-    "    \n",
-    "    # Obtain the values for t, x, and the excat solution:\n",
-    "    t, x, exact = data[\"t\"], data[\"x\"], data[\"usol\"].T\n",
-    "    \n",
-    "    # Process the data and flatten it out (like labels and features):\n",
-    "    xx, tt = np.meshgrid(x, t)\n",
-    "    X = np.vstack((np.ravel(xx), np.ravel(tt))).T\n",
-    "    y = exact.flatten()[:, None]\n",
-    "    \n",
-    "    return X, y\n",
-    "\n",
-    "def main():\n",
-    "\n",
-    "    def pde(x, y):\n",
-    "        \"\"\"\n",
-    "        Expresses the PDE residual of the heat equation.        \n",
-    "        \"\"\"\n",
-    "        dy_t = dde.grad.jacobian(y, x, i=0, j=1)\n",
-    "        dy_xx = dde.grad.hessian(y, x, i=0, j=0)\n",
-    "        return dy_t - a*dy_xx\n",
-    "\n",
-    "    # Computational geometry:\n",
-    "    geom = dde.geometry.Interval(0, L)\n",
-    "    timedomain = dde.geometry.TimeDomain(0, 1)\n",
-    "    geomtime = dde.geometry.GeometryXTime(geom, timedomain)\n",
-    "\n",
-    "    # Initial and boundary conditions:\n",
-    "    bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)\n",
-    "    ic = dde.icbc.IC(\n",
-    "        geomtime, lambda x: np.sin(n*np.pi*x[:, 0:1]/L), lambda _, on_initial: on_initial\n",
-    "        )\n",
-    "\n",
-    "    # Define the PDE problem and configurations of the network:\n",
-    "    data = dde.data.TimePDE(\n",
-    "        geomtime, pde, [bc, ic], num_domain=2540, num_boundary=80, num_initial=160, num_test=2540\n",
-    "    )\n",
-    "    net = dde.nn.FNN([2] + [20] * 3 + [1], \"tanh\", \"Glorot normal\")\n",
-    "    model = dde.Model(data, net)\n",
-    "\n",
-    "    # Build and train the model:\n",
-    "    model.compile(\"adam\", lr=1e-3)\n",
-    "    model.train(epochs=20000)\n",
-    "    model.compile(\"L-BFGS\")\n",
-    "    losshistory, train_state = model.train()\n",
-    "\n",
-    "    # Plot/print the results\n",
-    "    dde.saveplot(losshistory, train_state, issave=True, isplot=True)\n",
-    "    X, y_true = gen_testdata()\n",
-    "    y_pred = model.predict(X)\n",
-    "    f = model.predict(X, operator=pde)\n",
-    "    print(\"Mean residual:\", np.mean(np.absolute(f)))\n",
-    "    print(\"L2 relative error:\", dde.metrics.l2_relative_error(y_true, y_pred))\n",
-    "    np.savetxt(\"test.dat\", np.hstack((X, y_true, y_pred)))\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "4ji-gX9wybd-"
-   },
-   "source": [
-    "\n",
-    "In this code we will solve the heat equation using PINN implemented with the DeepXDE library.\n",
-    "\n",
-    "The equation is as follows:\n",
-    "\n",
-    "$\\frac{\\partial u}{\\partial t} = \\alpha \\nabla^2 u\\;$ .\n",
-    "\n",
-    "Where $\\nabla^2$ is the laplacian differential operator, $\\alpha$ is the thermal diffusivity constant and $u$ is the function (temperature) we want to approximate.\n",
-    "\n",
-    "In a unidimensional case we have:\n",
-    "\n",
-    "$\\frac{\\partial u(x, t)}{\\partial t}$ = $\\alpha \\frac{\\partial^2u(x,t)}{{\\partial x}^2}\\;$, $\\;\\;\\;\\; x \\in [0, 1]\\;$, $\\;\\;\\;\\; t \\in [0, 1]\\;$.\n",
-    "\n",
-    "With Dirichlet boundary conditions \n",
-    "\n",
-    "$u(0, t) = u(1, t) = 0\\;$ , \n",
-    "\n",
-    "and periodic (sinoidal) initial conditions:\n",
-    "\n",
-    "$u(x, 0) = sin(n\\pi x/L)\\;$, $\\;\\;\\;\\; 0 < x < L\\;$, $\\;\\;\\;\\; n = 1, 2, ...\\;.$\n",
-    "\n",
-    "This setup is a common problem in many differential equations textbooks and can be physically interpreted as the variation of temperature in a uniform and unidimensional bar over time. Here, the constant $\\alpha$ is the thermal diffusivity (a property of the material that the bar is made) and $L$ is the lenght of the bar.\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 1000
-    },
-    "id": "8QInMu-MxhIj",
-    "outputId": "b294ad3a-282b-4fcc-f6df-6c6fc2c16be3"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages\\skopt\\sampler\\sobol.py:246: UserWarning: The balance properties of Sobol' points require n to be a power of 2. 0 points have been previously generated, then: n=0+2542=2542. \n",
-      "  warnings.warn(\"The balance properties of Sobol' points require \"\n",
-      "c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages\\skopt\\sampler\\sobol.py:246: UserWarning: The balance properties of Sobol' points require n to be a power of 2. 0 points have been previously generated, then: n=0+82=82. \n",
-      "  warnings.warn(\"The balance properties of Sobol' points require \"\n",
-      "c:\\users\\saransh\\saransh_softwares\\python_3.9\\lib\\site-packages\\skopt\\sampler\\sobol.py:246: UserWarning: The balance properties of Sobol' points require n to be a power of 2. 0 points have been previously generated, then: n=0+162=162. \n",
-      "  warnings.warn(\"The balance properties of Sobol' points require \"\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Warning: 2540 points required, but 2550 points sampled.\n",
-      "Compiling model...\n",
-      "'compile' took 0.000510 s\n",
-      "\n",
-      "Training model...\n",
-      "\n",
-      "WARNING:tensorflow:AutoGraph could not transform <function main.<locals>.<lambda> at 0x00000250EE8918B0> and will run it as-is.\n",
-      "Cause: could not parse the source code of <function main.<locals>.<lambda> at 0x00000250EE8918B0>: no matching AST found\n",
-      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
-      "WARNING: AutoGraph could not transform <function main.<locals>.<lambda> at 0x00000250EE8918B0> and will run it as-is.\n",
-      "Cause: could not parse the source code of <function main.<locals>.<lambda> at 0x00000250EE8918B0>: no matching AST found\n",
-      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
-      "WARNING:tensorflow:AutoGraph could not transform <function main.<locals>.<lambda> at 0x00000250EE891700> and will run it as-is.\n",
-      "Cause: could not parse the source code of <function main.<locals>.<lambda> at 0x00000250EE891700>: no matching AST found\n",
-      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
-      "WARNING: AutoGraph could not transform <function main.<locals>.<lambda> at 0x00000250EE891700> and will run it as-is.\n",
-      "Cause: could not parse the source code of <function main.<locals>.<lambda> at 0x00000250EE891700>: no matching AST found\n",
-      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
-      "Step      Train loss                        Test loss                         Test metric\n",
-      "0         [6.96e-03, 3.61e-02, 7.18e-01]    [5.71e-03, 3.61e-02, 7.18e-01]    []  \n",
-      "1000      [8.44e-04, 5.63e-04, 2.80e-04]    [5.11e-04, 5.63e-04, 2.80e-04]    []  \n",
-      "2000      [1.77e-04, 2.28e-05, 4.37e-05]    [1.17e-04, 2.28e-05, 4.37e-05]    []  \n",
-      "3000      [1.01e-04, 4.83e-06, 1.49e-05]    [6.67e-05, 4.83e-06, 1.49e-05]    []  \n",
-      "4000      [8.28e-05, 3.98e-06, 7.70e-06]    [5.30e-05, 3.98e-06, 7.70e-06]    []  \n",
-      "5000      [5.54e-05, 1.01e-06, 4.36e-06]    [3.78e-05, 1.01e-06, 4.36e-06]    []  \n",
-      "6000      [4.39e-05, 6.96e-07, 3.08e-06]    [3.03e-05, 6.96e-07, 3.08e-06]    []  \n",
-      "7000      [4.17e-05, 3.12e-06, 2.98e-06]    [3.25e-05, 3.12e-06, 2.98e-06]    []  \n",
-      "8000      [2.92e-05, 4.24e-07, 1.89e-06]    [2.05e-05, 4.24e-07, 1.89e-06]    []  \n",
-      "9000      [2.80e-05, 2.08e-06, 2.02e-06]    [2.20e-05, 2.08e-06, 2.02e-06]    []  \n",
-      "10000     [2.00e-05, 3.02e-07, 1.16e-06]    [1.45e-05, 3.02e-07, 1.16e-06]    []  \n",
-      "11000     [2.82e-05, 3.98e-06, 2.57e-06]    [1.84e-05, 3.98e-06, 2.57e-06]    []  \n",
-      "12000     [1.43e-05, 2.19e-07, 7.88e-07]    [1.04e-05, 2.19e-07, 7.88e-07]    []  \n",
-      "13000     [3.12e-04, 9.89e-05, 3.64e-05]    [2.53e-04, 9.89e-05, 3.64e-05]    []  \n",
-      "14000     [7.01e-05, 1.15e-05, 1.65e-06]    [6.12e-05, 1.15e-05, 1.65e-06]    []  \n",
-      "15000     [9.41e-06, 1.92e-07, 4.52e-07]    [6.77e-06, 1.92e-07, 4.52e-07]    []  \n",
-      "16000     [8.37e-06, 1.79e-07, 3.90e-07]    [6.01e-06, 1.79e-07, 3.90e-07]    []  \n",
-      "17000     [7.54e-06, 1.78e-07, 3.45e-07]    [5.39e-06, 1.78e-07, 3.45e-07]    []  \n",
-      "18000     [6.87e-06, 1.74e-07, 3.06e-07]    [4.87e-06, 1.74e-07, 3.06e-07]    []  \n",
-      "19000     [6.42e-06, 2.46e-07, 3.13e-07]    [4.66e-06, 2.46e-07, 3.13e-07]    []  \n",
-      "20000     [8.23e-06, 8.63e-07, 5.22e-07]    [5.22e-06, 8.63e-07, 5.22e-07]    []  \n",
-      "\n",
-      "Best model at step 19000:\n",
-      "  train loss: 6.98e-06\n",
-      "  test loss: 5.22e-06\n",
-      "  test metric: []\n",
-      "\n",
-      "'train' took 143.634160 s\n",
-      "\n",
-      "Compiling model...\n",
-      "'compile' took 0.016476 s\n",
-      "\n",
-      "Training model...\n",
-      "\n",
-      "Step      Train loss                        Test loss                         Test metric\n",
-      "20000     [8.23e-06, 8.63e-07, 5.22e-07]    [5.22e-06, 8.63e-07, 5.22e-07]    []  \n",
-      "20006     [5.87e-06, 1.75e-07, 2.55e-07]    [4.10e-06, 1.75e-07, 2.55e-07]    []  \n",
-      "\n",
-      "Best model at step 20006:\n",
-      "  train loss: 6.30e-06\n",
-      "  test loss: 4.53e-06\n",
-      "  test metric: []\n",
-      "\n",
-      "'train' took 2.057389 s\n",
-      "\n",
-      "Saving loss history to D:\\OpenSource\\python\\deepxde\\examples\\loss.dat ...\n",
-      "Saving training data to D:\\OpenSource\\python\\deepxde\\examples\\train.dat ...\n",
-      "Saving test data to D:\\OpenSource\\python\\deepxde\\examples\\test.dat ...\n"
-     ]
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEHCAYAAABCwJb2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8xUlEQVR4nO3dd3xUVfr48c8zk95JCC0BEgjSAtIREEHsIKuy6uJasPe2/uzurq7rfl1317q6KrsCFlZF1wKiIhakSpVeQxIgjTRIQkL6+f1xJxhCAikzmUnmeb9e85qZm1ueuUmee+acc88RYwxKKaW8g83dASillGo9mvSVUsqLaNJXSikvoklfKaW8iCZ9pZTyIpr0lVLKi/i4O4CT6dixo4mLi3N3GEop1aasX78+1xgTXd/PPDrpx8XFsW7dOneHoZRSbYqI7GvoZ62W9EUkGPgXUA4sMcbMba1jK6WUsrSoTl9EZolItohsrbP8QhHZJSJJIvKoY/E04GNjzC3Ar1pyXKWUUs3T0obcOcCFtReIiB14DbgIGABcJSIDgFjggGO1qhYeVymlVDO0qHrHGLNUROLqLB4FJBljkgFE5APgEiANK/FvRHsNKeXVKioqSEtLo7S01N2htGkBAQHExsbi6+vb6G1cUacfwy8lerCS/WjgFeBVEZkCLGhoYxG5FbgVoEePHi4ITynlbmlpaYSGhhIXF4eIuDucNskYQ15eHmlpacTHxzd6u1ZryDXGFAM3NGK9mSKSCUz18/Mb7vrIlFKtrbS0VBN+C4kIUVFR5OTkNGk7V1SzpAPda72PdSxrNGPMAmPMreHh4U4NTCnlOTTht1xzzqErkv5aoI+IxIuIHzAdmN+UHYjIVBGZWVBQ0KwA1n/7ASvn/qVZ2yqlVHvW0i6b7wOrgL4ikiYiNxljKoG7gUXADmCeMWZbU/bb0pJ+1faF9NvzerO2VUq1f3l5eQwZMoQhQ4bQpUsXYmJijr0vLy8/6bbr1q3j3nvvbdLx4uLiyM3NbUnITtPS3jtXNbD8S+DLluy7JUxkbyLz53MoL5sOUZ3cFYZSykNFRUWxceNGAJ566ilCQkJ48MEHj/28srISH5/60+OIESMYMWJEa4TpEh7ZdbKl1TuBXfoAkJW89RRrKqWU5frrr+f2229n9OjRPPzww6xZs4YxY8YwdOhQxo4dy65duwBYsmQJF198MWBdMG688UYmTpxIr169eOWVV055nBdeeIHExEQSExN56aWXACguLmbKlCmcfvrpJCYm8uGHHwLw6KOPMmDAAAYPHnzcRaklPHLsHWPMAmDBiBEjbmnO9lE9BgJQkL4TRk5yZmhKKSf704JtbM8odOo+B3QL48mpA5u8XVpaGitXrsRut1NYWMiyZcvw8fHh22+/5fHHH+d///vfCdvs3LmTH374gaKiIvr27csdd9zRYL/59evXM3v2bFavXo0xhtGjRzNhwgSSk5Pp1q0bCxcuBKCgoIC8vDw+/fRTdu7ciYhw+PDhJn+e+rTLkn7nnn2pMkJlTpKTI1NKtWdXXHEFdrsdsBLvFVdcQWJiIr/73e/Ytq3+pskpU6bg7+9Px44d6dSpEwcPHmxw/8uXL+eyyy4jODiYkJAQpk2bxrJlyxg0aBCLFy/mkUceYdmyZYSHhxMeHk5AQAA33XQTn3zyCUFBQU75jO2ypO/jH0iGrRO+h1OcHJlSytmaUyJ3leDg4GOv//CHP3D22Wfz6aefkpqaysSJE+vdxt/f/9hru91OZWVlk4972mmnsWHDBr788kt+//vfc8455/DHP/6RNWvW8N133/Hxxx/z6quv8v333zd533V5ZEnfGQ4FdCfi6H53h6GUaqMKCgqIiYkBYM6cOU7Z5/jx4/nss88oKSmhuLiYTz/9lPHjx5ORkUFQUBDXXHMNDz30EBs2bODIkSMUFBQwefJkXnzxRTZt2uSUGDyypC8iU4GpCQkJzd7H0bA4+mYtpKqqGru93V7blFIu8vDDDzNjxgyeeeYZpkyZ4pR9Dhs2jOuvv55Ro0YBcPPNNzN06FAWLVrEQw89hM1mw9fXl9dff52ioiIuueQSSktLMcbwwgsvOCUGMcY4ZUeuMGLECNPcSVR+nvcsQ7f/lQM3baJ79zjnBqaUapEdO3bQv39/d4fRLtR3LkVkvTGm3n6l7bYIHNytHwA5qdvdHIlSSnmOdpv0o+OsK9+RjJ1ujkQppTyHRyb9lnbZBIjo2psK7FTn7XViZEop1bZ5ZNJ3xiibYvfloL0rgYXabVMppWp4ZNJ3loLAHkSWHTj1ikop5SXaddKviIine3UmR0pPPmqeUkp5i3ad9H2i+xAgFaSlar2+UuoXLRlaGaxB11auXFnvz+bMmcPdd9/t7JCdpt3enAUQHtsffoa8A9uhn/YJVkpZTjW08qksWbKEkJAQxo4d66IIXccjS/rOmi6xpttmaeZuZ4SllGrH1q9fz4QJExg+fDgXXHABmZmZALzyyivHhjeePn06qampvPHGG7z44osMGTKEZcuWNbjP1NRUJk2axODBgznnnHPYv98aGuajjz4iMTGR008/nbPOOguAbdu2MWrUKIYMGcLgwYPZs2ePSz6nR5b0nSWgQ3dK8UMOJbs7FKVUQ756FLK2OHefXQbBRX9t9OrGGO655x4+//xzoqOj+fDDD3niiSeYNWsWf/3rX0lJScHf35/Dhw8TERHB7bff3qhvB/fccw8zZsxgxowZzJo1i3vvvZfPPvuMp59+mkWLFhETE3NsyOQ33niD++67j6uvvpry8nKqqqpacgYa1K6TPjYbOb4xBB/RbptKqYaVlZWxdetWzjvvPACqqqro2rUrAIMHD+bqq6/m0ksv5dJLL23SfletWsUnn3wCwLXXXsvDDz8MwLhx47j++uu58sormTZtGgBjxozhL3/5C2lpaUybNo0+ffo46dMdr30nfaAouCfRh3ZjjGnWzPFKKRdrQoncVYwxDBw4kFWrVp3ws4ULF7J06VIWLFjAX/7yF7Zsafm3kjfeeIPVq1ezcOFChg8fzvr16/ntb3/L6NGjWbhwIZMnT+bNN99k0iTnTwLlkXX6zlTdoRfdOcjBw8XuDkUp5aH8/f3Jyck5lvQrKirYtm0b1dXVHDhwgLPPPpvnnnuOgoICjhw5QmhoKEVFRafc79ixY/nggw8AmDt3LuPHjwdg7969jB49mqeffpro6GgOHDhAcnIyvXr14t577+WSSy5h8+bNLvmsHpn0nTEMQw3/Ln3xlSrSU3c5ITKlVHtks9n4+OOPeeSRRzj99NMZMmQIK1eupKqqimuuuYZBgwYxdOhQ7r33XiIiIpg6dSqffvrpKRty//nPfzJ79mwGDx7Mu+++y8svvwzAQw89xKBBg0hMTGTs2LGcfvrpzJs3j8TERIYMGcLWrVu57rrrXPJZ2+3QyjVyty+h47xL+G74a5wz9RonRaaUagkdWtl5dGjlOqJ6DACgPNs13Z+UUqotafdJX4KjKSYIX+22qZRS7T/pI0KefyxhJTpfrlKexJOrltuK5pzD9p/0gZLQeLpWpVNW6ZqbHZRSTRMQEEBeXp4m/hYwxpCXl0dAQECTtmv3/fQBiOpFt5xvSM4+RJ9uHd0djVJeLzY2lrS0NHJyctwdSpsWEBBAbGxsk7bxiqQf3LUf9l2GrNSd9Ol2prvDUcrr+fr6Eh8f7+4wvFKrVe+ISC8ReUtEPm6tY9aI7GF1ZypK1/lylVLerVFJX0RmiUi2iGyts/xCEdklIkki8ujJ9mGMSTbG3NSSYJsruOtpAFTlJrnj8Eop5TEaW70zB3gVeKdmgYjYgdeA84A0YK2IzAfswLN1tr/RGJPd4mibK7ADhbZw/At04DWllHdrVNI3xiwVkbg6i0cBScaYZAAR+QC4xBjzLHCxU6N0gkMBPehQovPlKqW8W0vq9GOA2lk0zbGsXiISJSJvAENF5LGTrHeriKwTkXXObNkvC4+ju8kgv1jny1VKea9Wa8g1xuQZY243xvR2fBtoaL2ZwJ+ADX5+fk47vr1jAl3kEKkZB522T6WUamtakvTTge613sc6lrWYs6ZLrC00ph8Aufu1B49Synu1JOmvBfqISLyI+AHTgfnOCMqZQyvXiOxuddss1vlylVJerLFdNt8HVgF9RSRNRG4yxlQCdwOLgB3APGPMNmcE5YqSvk90AgCSp902lVLeq7G9d65qYPmXwJdOjQirpA9MTUhIcN5O/YI5ZI8i8Eiq8/aplFJtjEcOuOaKkj5AYVBPossOUFWtgzwppbyTRyZ9V6mM6EVPySLtUIm7Q1FKKbfwyKTvioZcAL/OfYiSIvanZzh1v0op1VZ4ZNJ3VfVORKzVgyd//w6n7lcppdoKj0z6rhLSrS8ApTpfrlLKS3lk0ndV9Y5ExlOFDZ/8vU7dr1JKtRUemfRdVb2Djz+HfTsTUrLPuftVSqk2wiOTvisVh/Ska2U6R8oq3R2KUkq1Oq9L+iayN3GSRUr2EXeHopRSrc4jk76r6vQBArqcRpgcJT1dq3iUUt7HI5O+y+r0gQ6x1mibBem7nL5vpZTydB6Z9F3Jr7M1X25ljnbbVEp5H69L+oT3oBI7voeT3R2JUkq1Oo9M+q6s08fuw2H/GMKPHsAYHXhNKeVdPDLpu7JOH6A0LI4eJoOswlKX7F8ppTyVRyZ9V5OOfYiTg6RkF7k7FKWUalVemfRDup5GoJSTmZbi7lCUUqpVeWXSD3N02zySoZOkK6W8i1cmfYmypmE0uTpfrlLKu3hl0ie0G+XiT2BRqrsjUUqpVuWRSd+lXTYBbDYKArvTsewAZZVVrjmGUkp5II9M+q7usglQEdGLOMliX57Ol6uU8h4emfRbg090Aj0km5SDh90dilJKtRqvTfphMf3wlSqy07QxVynlPbw26Qc4Bl4ry9rt5kiUUqr1eG3SJ6q39azz5SqlvIj3Jv3gaEptwQQf0clUlFLew3uTvghFwT2IqUonv7jc3dEopVSraNWkLyKXisi/ReRDETm/NY9dn+oO1ny5yTk6X65Syjs0OumLyCwRyRaRrXWWXygiu0QkSUQePdk+jDGfGWNuAW4HftO8kJ3Hv3MfYiSX1IOH3B2KUkq1iqaU9OcAF9ZeICJ24DXgImAAcJWIDBCRQSLyRZ1Hp1qb/t6xnVuFxvTDLoZD6dqDRynlHXwau6IxZqmIxNVZPApIMsYkA4jIB8AlxphngYvr7kNEBPgr8JUxZkN9xxGRW4FbAXr06NHY8JrF3rEPAOUHd2Ndt5RSqn1raZ1+DHCg1vs0x7KG3AOcC1wuIrfXt4IxZqYxZoQxZkR0dHQLwzuFqF4A+BzWcfWVUt6h0SV9ZzDGvAK8cqr1RGQqMDUhIcG1AQV2oMQngvCj+6iqNtht4trjKaWUm7W0pJ8OdK/1PtaxrEVaY8C1GiUhPelpskg7pAOvKaXav5Ym/bVAHxGJFxE/YDowv6VBuXxo5dqiEoi3ZZKcU+z6YymllJs1pcvm+8AqoK+IpInITcaYSuBuYBGwA5hnjNnW0qBas6Qf1PU0usgh9mXluPxYSinlbk3pvXNVA8u/BL50WkS0Yp0+ENS1LwCFGbuAgS4/nlJKuZNHDsPQmiV9Iq2B16pzdIhlpVT755FJv1VFWt02/Qq126ZSqv3zyKTfqg25/iEc8YumU3kaR8oqXX88pZRyI49M+q1avQOUhccTJ1mkaA8epVQ755FJv7XZOyYQL5kk5+pom0qp9s0jk36rVu8Awd36EiVFpGdmtsrxlFLKXTwy6bd29Y5vtDXwWkmmjraplGrfPDLpt7oo634Ak6fdNpVS7ZtHJv3Wrt6hQxzV2Ag+kooxpnWOqZRSbuCRSb+1q3fw8acksAsx1ZlkFZa2zjGVUsoNPDLpu0NFRG/iJVO7bSql2jVN+g5+nRKIkyz26iTpSql2TJO+Q1DXvoTJUbIyD5x6ZaWUaqM8Mum3ekMuII4ePGVZe1rtmEop1do8Mum3ekMuQJQ12qb9cHLrHVMppVqZRyZ9twjvQZXYiSjZR1lllbujUUopl9CkX8PuQ0lwD3pKFvvydL5cpVT7pEm/FtOhlzXwmvbgUUq1U5r0awno2pc4Ocje7CJ3h6KUUi6hSb8Wv+gEAqWc/KxUd4eilFIu4ZFJ3x1dNoFjPXgqsrXbplKqffLIpO+WLptwbLRNvwKdL1cp1T55ZNJ3m9BuVNr86VyRzn7twaOUaoc06ddms2Eie9HblsV/lutNWkqp9keTfh2+0QkkBuTywdoDZBfpMMtKqfZFk35dUQlEV2Ziqip4a7nW7Sul2hdN+nV1GYxUV/BArwzeW7WPwyXl7o5IKaWcptWSvoj0F5E3RORjEbmjtY7bZP0uhtCuXGc+p7i8irdX7nN3REop5TSNSvoiMktEskVka53lF4rILhFJEpFHT7YPY8wOY8ztwJXAuOaH7GI+fnDGnQRnrOTmXvnMXplCcVmlu6NSSimnaGxJfw5wYe0FImIHXgMuAgYAV4nIABEZJCJf1Hl0cmzzK2Ah8KXTPoErDL8e/MO5y+9LDpdU8N/V+90dkVJKOUWjkr4xZimQX2fxKCDJGJNsjCkHPgAuMcZsMcZcXOeR7djPfGPMRcDVzvwQThcQBiNvpEPqV0zrWcq/lyVTWqHDLSul2r6W1OnHALXnFkxzLKuXiEwUkVdE5E1OUtIXkVtFZJ2IrMvJyWlBeC00+g6w+/Jw2GKyi8r4eH2a+2JRSiknabWGXGPMEmPMvcaY24wxr51kvZnGmBHGmBHR0dGtFd6JQjvD6VfROfkTJsZU88aPe6msqnZfPEop5QQtSfrpQPda72Mdy1rMbQOu1TX2XqSqnCc7LSft0FHmb8pwbzxKKdVCLUn6a4E+IhIvIn7AdGC+M4Jy24BrdXVMgP4XE5fyPkM72/nXkr1UVxv3xqSUUi3Q2C6b7wOrgL4ikiYiNxljKoG7gUXADmCeMWabM4LymJI+wLj7kdIC/tx9PUnZR/hme5a7I1JKqWYTYzy35DpixAizbt06d4cBs6dg8pM5r+plAgODmH/3OETE3VEppVS9RGS9MWZEfT/zyGEYPKqkD3Dm/UhRBs8k7GJLegFL9+S6OyKllGoWj0z6HlOnXyPhXOg0kFGZ79EtzI/Xfkhyd0RKKdUsHpn0PY4IjLsPW85O/tQ/gzUp+axNrXuvmlJKeT6PTPoeV70DkDgNwrszKf99IoO1tK+Uaps8Mul7XPUOgN0XxtyF/cAqHh9UxJJdOWxN96CLklJKNYJHJn2PNfRaCIjgkuKPCPX34V9LtLSvlGpbPDLpe2T1DoB/CIy6Fd89X3H/UMNXW7NIyj7i7qiUapSMw0cpKKlwdxjKzTwy6Xtk9U6NUbeCjz9XV83H38fG60v2ujsipU7JGMP0mT9xw5w1ePK9Ocr1PDLpe7SQaBh6DQHbP+KWIYF8tjGdA/kl7o5KqZPam1PM/vwSNuw/zFdb9a5yb+aRSd9jq3dqjLkbqiu5zf8bbAIzlya7OyKlTmrZnhze8H2RZ4M/4Lmvd1JeqSPGeiuPTPoeXb0DEBkPAy4lZMu7/Pb0CD5cd4DsolJ3R6VUgzbs3Mv59nVMr1pAUP4O5q7WuZ+9lUcm/TZh3H1QVsh9EcuprKrmrWUp7o5IqXqVVVZhS12ODQM2H/4WNo9Xvt1NwVFt1PVGmvSbq9sQ6DWRyM1vcWliR977aR+HS8rdHZVSJ1i/7xAjzWYqfYKRc59iUNnPDClfq50QvJQm/ZYYdx8cyeKhbpsoLq/i7ZX6lVl5nmV7chln3wY9x1m9zyJ782zwPN5ekUT64aPuDk+1Mo9M+h7fkFuj19nQZTBdt87kvH7RzF6ZQnFZpbujUuo4O3ZuJ16y8EmYCD5+cP6f6VK+jyvlO55ftMvd4alW5pFJ3+Mbcms4BmIjbw+P9U7mcEkF/129391RKXVM7pEyOub8ZL2Jn2A9950MPc/kYf9P+XbjHh1OxMt4ZNJvUwZcChE96bXrP4zrHcnMZcmUVlS5OyqlAFiRlMs421YqA6Kg0wBroQhc8AzBlYf4nf8Cnv1qh96w5UU06beU3QfG3gNpa3ls4GFyisq47q01HCzULpzK/ZbuyuFM+zZsvSeCrda/e7ehcPpVXGv7in17d7Jkd47bYlStS5O+Mwy5GoKiSEydw0u/GcLWjAImv7yMZXv0H0m5jzGGtD0/E81hbL0mnLjCpD9gt9l5Kuhj/vrlTqqqtbTvDTTpO4NfEIy6DXZ/zaXdCph/9ziiQvy4btYaXly8W/+ZlFvsOlhEv6M/W2/qS/rhMcjYuzm3ahmB2T/z8foDrRugcgtN+s4y6hbwDYKV/yShUyif3TWOy4bG8PJ3e5gxaw25R8rcHaHyMst2O+rzw3pAh7j6Vxp3PyakM38NeZ/nF+2ipFx7n7V3Hpn020yXzdqCImHYdbBlHuTsJsjPh+evOJ2//Xowa1PzmfzyMlYn57k7SuVFlu/OYpx9h9VVsyH+IcjZT9CvYgfDS5bx76V6Z3l755FJv8102axr7D3gHwpvnQtJ3yEiXDmyO5/eOY5gfx9++5/VvL5kL9Va3aNcrLSiipLUdQRTAr0mnnzloddAp4E8HTSPWUt36jhS7ZxHJv02KzwWbvkBwmJh7uWw8lUwhgHdwph/9zguTOzCc1/v5OZ31nGoWIdsUK6zJiWfkWaL9Sa+nvr82mx2uOAZoiszmV79FS99u8f1ASq30aTvbJHxcNM30O9i+OYJ+PQ2qDhKaIAvr141lD9fMpDle3K5+J/L+Xn/IXdHq9qpZXtyGG/fRnWngRDc8dQb9J4ECedxv//nfLN2G0nZRa4PUrmFJn1X8A+BK96Gs5+AzR/C7IugIB0R4doxcXx8xxhE4Mo3VzFreYreGKOcbvXudIbbdmM7VdVObef/mYDqEn7n+yl//Wqny2JT7qVJ31VsNpjwMEz/L+TugZkTYf9qAAbHRrDwnvFM7NuJp7/Yzh3vbaCwVIe5Vc6RXVhKSM4G/Kiov6tmQzr1R4Zfz3TbYpJ3buQn7XjQLmnSd7V+U+Dmb63S/5wpsH4OAOFBvsy8djhPTO7P4h0HmfrP5ToGinKKpXtyOdO2FSM+0HNs0zae+Dg230CeCvyQ//tyh3Y6aIdaNemLSLCIrBORi1vzuG7XqT/c8j3Ej4cF98HCB6GqAhHhlrN6Me+2MyivrGba6yt57JPNLNycSb429KpmWrYnhwk+2yF2hNWbrClCopHxD3BW9VqCMlaxYHOGa4JUbtOopC8is0QkW0S21ll+oYjsEpEkEXm0Ebt6BJjXnEDbvMAO8NuPrG6da/8N71wKxbkADO8ZycJ7xzM5sQtfbMrkrv9uYPgzi5nyyjKe/XIHS3fncLRcB3FTp1Zdbdi4ex/9SUaaUrVT2xl3YsJjeSbwff7x9Q7KKvVvrz3xaeR6c4BXgXdqFoiIHXgNOA9IA9aKyHzADjxbZ/sbgdOB7UBAy0Juw+w+cP4z0GUwzL/Hquef/l/oOpjIYD9emj6UyqpqNqcXsGJPLiv25jJ7RSpvLk3Gz25jWM8IzkzoyNiEjgyOCcfHrrVz6njbMwvpW7oRm1/1qbtqNsQ3EDnnKRI+uZlRRxfzzspe3HJWL+cGqtxGGttzRETigC+MMYmO92OAp4wxFzjePwZgjKmb8Gu2/wsQDAwAjgKXGWOqT3bMESNGmHXr1jXuk7Q16Rvgw2ugJB8ufQ0Sf13vaiXllaxNPcTKpFyWJ+WyLaMQgFB/H87oHcW43lGc2acjvaNDEJHW/ATKA/1rSRJB3z7GjIBlyKP7rUlTmqO6Gv5zDvkH93Nh1Yt88/CFRAQ1c1+q1YnIemPMiPp+1tiSfn1igNojNKUBoxta2RjzhCOY64HchhK+iNwK3ArQo0ePFoTn4WKGWTdyzbsOPr4RsrbCpN9bN8rUEuTnw4TToplwWjQA+cXlrNqbx/KkXFYk5bJ4+0EAOof5c1FiV24eH09sh6BW/zjKMyzbnctzfjuQnmObn/DB6n12wf8ROftCpld+zqvfJ/D7iwc4L1DlNi1J+s1ijJlzip/PBGaCVdJvjZjcJrQzzFgAXz0Ey1+Ag1th6isQ1rXBTSKD/ZgyuCtTBlvrHMgvYUVSLsv25DJ39T7e+2kflw6N4fYJvUnoFNJan0R5gJLySg7sS6KH7wHodWvLd9hzDPT/FXftWsikVecwY2wc3SO1QNHWtaRSOB3oXut9rGNZi7XJAdeay8cPpr4MU16Avd/DC/3gP+fC8het/v2n0D0yiOmjevDa1cP48aGzuXZMT77YnMF5L/7InXPXazdQL/JTch4jjaOvRXMbces670/4Ucn99nn8XefTbRdakvTXAn1EJF5E/IDpwHxnBNVmB1xriZE3wZ0/wdm/h6oK+PYpeHUEvDoSFj8JB9Za9awn0S0ikCenDmTFI5O4a2ICyxzDPcyYtYY1Kfmt8zmU2yzdnctZPtswgZHQeZBzdhrZCxl9G5fblpC0ZRUH8kucs1/lNo1qyBWR94GJQEfgIPCkMeYtEZkMvITVY2eWMeYvTglKZCowNSEh4ZY9e7x08KeCNNj5JexaCKnLoboSQjpbk1r3u9jq8+/jf9JdFJZW8N5P+3hrWQp5xeWMjOvAnWcnMPG0aG30bYfO+ccPzDt6C1F9x8KV75x6g8Y6eojql4eyvKQ7S0e9qXX7bcDJGnIb3XvHHdp1752mOHoI9iyGnV/Anm+hohj8QqHPedYdv33Og4CGvxUdLa9i3roDvPnjXjIKShnYLYy7zk7ggoFdsNs0+bcH6YePcs1zc/nB//9ZVYUjb3LuAZY9D989zVTzEu8/fi0h/q3eHKiaoM0lfS3pn0RFKaT8aF0Adn0FxTlg87VK/omXW10/feu/FaK8sprPNqbzxpK9JOcW0ys6mDsm9ObSoTH4ap//Nu2DNfvZ+vkLPOM7G+7ZAFG9nXuAI9mY5/vzn4rz8bno/7hhXLxz96+cqs0l/Rpa0j+F6ipIWws7F8KOBXAoBYKjYcRNMOJGq3dQPaqqDV9vzeK1H5LYnllITEQg143pycS+nTits/b3b4vumruBy5MeY2JYGnL/VnDF73DeDIp2fMe0wLf4+sHz9VuiB2tzSV9L+s1gjPUN4KfXYffXYPezSv5n3A5dT29gE8OS3Tn864ck1qZaY/t3DPFjTO+OjOsdxbiEjtpFrw2oqjaMePprVthvIWjQJdbNfq6w9wd491LuK7+Ti6+5n/MG1F+oUO7nqpuzXMYYswBYMGLEiFvcHUubIWJNi9drIuQmwZo34ee5sOm/0PNMK/n3nXzczV8iwtl9O3F2306kHSphZVIeK/fmsmJvHgs2WQNtdY8MZFzvjozpHcXY3h2JDj1547FqfVvSC4gpSyLIv8h5XTXrEz8B0yGeGw4v4bnll2jSb6M8MumrFuqYAJP/bk3i8vO7sHqmNeRDRE8YfZs1J2qdht/YDkFcOTKIK0d2xxhDUvYRViRZF4CFWzL5YK1183XfzqGMTYhiXO+OjOoVSViArzs+oapl6e4czrRvs97En+W6A9lsyPDrGfLtk+SkbGJ7xgAGdAtz3fGUS2j1jjeoqrS6fv70OuxfZfX8GXoNjL4VIk89kFZVtWFregEr9uayMimPtan5lFVWY7cJg2LCGdGzA/6+NoyBmr+mmj8rwy8Lf/mZOfZzu104r39nhvfsoG0JzXTFGyt5LO8JhkWUwF2rXXuwIzmYF/rzbuW5bB70GP+4ov6qQ+Veba5Ov4Y25LpA+gZY/QZs/cTq+9/3IjjjDogb3+jGv9KKKn7ef9iqCkrKZWt6IdWOvyMREBz7qfUkx15LrfWgvKqaiirDkO4R3DK+FxcM7KyjhzZBUWkFo57+ks0Bt+A78ga46DnXH/SjGyjZuZjRpa/x/aMXapWfB9Kkr05UmAnr3oJ1s6AkD6L6QO+zreqBnuMgKLJVwigpr+R/69N4a3kKqXklxHYI5MZx8Vw5srv2BW+Eb7ZlMWvuu3zg9wxMfx/6TXb9QVOWwttT+V35HfScdCP3n3ua64+pmkSTvmpYxVHY8hFs+xT2/wQVJYBAl0RrPPa48dbAWye5+atFqirA7ktVteHbHQf599Jk1u07RGiAD78d3YMbxsbTJdx7p2A4ld9/toXYn1/gNttnyCOprvs91WYM/HM4e4oDuKrqT6x4dBL+PvZTb6daTZtL+lqn7yaV5ZC+HlKXWaW5A2ugqgzEBl2HWN8C4sdDjzHgF9y4fVZXw5EsyE+x7iM4lHr865I86D8Vznv6WPvCz/sP8Z9lKXy1NRObCFNP78bN4+MZ2M2LxmJqpAl//4G3Kh8noWMQ3PJd6x14xSuw+A+cV/Y3brt8CpcPj229Y6tTanNJv4aW9N2sohTS1kCK4yKQvs5qB7D5QMxw6yIQNx66DYGig1Yiz3ck85rXh/dBZekv+xQ7hMdCZDx0iAOfQNjwDlRXWG0L4x+EAKtHyIH8EmatSOHDtQcoKa9ibO8objmrV5seO6i0ooqNBw4zKi4SWwtvbtqXV8yUv3/J5sDbsJ15P5zzR+cE2RjFeZgX+vG5/XxmBt/OwnvPbLO/k/ZIk75yjvJiqwooZan1bSDjZ6hvLhzf4F+Seoc4x2vH+4geYK/TzbMwE75/BjbOhaAoazKZYdcdu6eg4GgF76/Zz5wVqWQVltKnUwg3j4/nkiExBPg2vVrBGENltcHHJq2WqCqqqvl4fRqvfLeHzIJSJg/qwgtXDmlW/DXe+2kf389/h1l+/7DmZXBld836fHwT5TsXMfjIK8y+ZQJjeke17vFVgzTpK9coLbS6gB7cBmHdrMQeGW8NBdGcZJrxM3z9OOxfCZ0T4YK/WDebOZRXVrNwSwb/XprC9sxCOob40a9LGBVV1VRWGyqrqimvsp4rqw3lldVUVldTWWUor7KeK6ut3kIAXcMDuHZMT64a2YMOwa6ZCrCq2rBgUwYvfrubfXklDO0Rwci4SGYuTWZUXCQzrxve7GkIb31nHefue5Er5FvkkX0NjrnkMqnLYc4UnpS7yIifxr+vqzfHKDfQpK/aDmNg++ew+A9weL91F/H5zxw3gJgxhlV783h7VSo5RWX42G342gVfuw0fm/X62DKbDR/Hz35ZbsPHJqxOyWNFUh4BvjYuGxrLDePiOK1zqJM+hmHRtoO8sHgXuw8eoX/XMB48/zQm9euEiLBgUwb/b94mekQFMeeGkU2e4rKiqpphTy9mceCjdOnWA6773ClxN4kx8NooMsr8GZf7GEsenEjPqEa29SiXanNJXxtyFRWlsPp1WPq81SYw+jY46yEIjHDqYXZmFTJnRSqf/pxOWWU1ZyZ05IZxcZzdt1Oz6tyNMSzdk8vz3+xic1oBvaKDeeC805ic2PWE/a3am8et764j0NfO7BtGNqmhel1qPre/8TXrAu6Ac56E8Q80OVanWPUaLHqcKRXPMeqM8Tw5daB74lDHaXNJv4aW9BVFB+GHZ2DDuxDYAc5+HIbfAPYm9uE3BoqyIH8v5O21nkM6W/vyCyK/uJz31+zn3VX7yCosJS4qiBlj47hiROPvF1iTks8/Fu1iTWo+sR0Cue+cPlw2NOakN5vtPljEjFlrKCqt5PVrhjG+T3SjjvXCN7vY9+M7vOz7KtzyvdWw7g4l+fB8P5aFTeaO/KtY9dgkQnVoDrfTpK/avszNsOhxqwE5up9V359w7vHrGGPNL1CT1I89J0N+sjX5TA2br9VjKKQLTHgIhs0Auy8VVdV8vTWL2StS2LD/MKH+PlwxojvXj42jR1T9VTCb0w7zj292s3R3Dp1C/blnUgK/GdkDP586yb4kH1a/aQ2Ffd5Tx+LPLDjKDbPXkpR9hL9dPphpw07d/fHS11Zwd9FLnMtqeDjluIH0Wt3/bqFq51ckFr3MgxcP46Yzdax9d9Okr9oHY6yE+c3vrS6hfc637h84luCToazwl/VtPtYgc1G9IbK39VzzOjzWug/huz9ZjdEd4q1eQwOngc1K1hsPHGb2ihQWbs6kyhjO6deZG8fFMaZ3FCLCrqwiXli8i0XbDtIhyJc7Jvbm2jPiCPSrk4CPZMOqV2HtW1B+BII7WfcnTP4bjLwZsKa2vO2d9axKzuOhC/py58TeDfYsOlxSzrA/f8PPYQ8SHj8cps91xdluvH0rYfZF/Cvsft6vnMCSB8/WsfbdTJO+al8qy2DNTPjxb1YSjejxS1I/9tzLSvinqgYyxpqK8rs/wcGt1oTi5z5plcIdSfdgYSnv/bSPuav3k19cTr8uofSKDuarrVmE+Plw8/he3Hhm3InVGgVp1k1MG96GqnLrgjL+ASvej2+CPYtg9O1wwf+BzU5ZZRUPfbSZ+ZsyuOaMHvzpV4n1Js8vt2Ty1/9+xVL/38Hkf8AoN49Abgy8NprDVf4MyXyUN64ZzoWJXdwbk5fTpK/ap8oyQMDHCd0tq6th6/+s9oNDqdBjrJX8e5xxbJXSiirmb8pg9opUUnKPcP3YeG47q9eJ3T3zk2H5S7Dxv4CBwdPhzN9ZQ14fO14VfPMH+Ok16xvL5bPAP5TqasNzi3by5o/JnNu/M/+8augJ3xwe/d9mAre8y5PMhLvWQrQHjH3z0+vw9aNc7/cCJVEDmHfbGHdH5NU06SvVWJXlVsl86d/hyEE47SI45w/Q+ZdeKebYiKJ1SuHZO2H5C9ZYRjZfGHYtjLvPKtk3ZO1b8OVDVjvFbz84tu7bK1N5asE2hnSP4K0ZI4l0XFiMMZz53A+8bH+ZEbZd8MAO10yN2FRHD8Hz/djeeSqT917GF/ecSWKMDpvhLidL+h45hq2ITBWRmQUFBe4ORXkbHz+ruuTen61hDfathNfHwSe3WsNKYCX74xJ+5ib48Fr41xnWXMVn3An3bYIpz5884QOMvAmu+diqCvr3OZBmFXJmjI3j9auHsT2jkF+/vpJ9eVYjdHJuMRmHi0ks32QNiOcJCR+snlUDL6N/zldE+1Uwa0WKuyNSDfDIpG+MWWCMuTU8XEsKyk38gmH8/4P7NsK4e60bxl4daZXKj2Rb6xxYA3OvhDfPguQl1vr3b7V6FoV1bfyxek+Cm74B30CYM8Wa6wC4MLErc28ezaGScn79+ko2HTjMst059JMDBFQcOu5uZY8w/Aak/Ah/iNvBgk0ZZBeVnnob1eq0ekepxijMsBqON7wDPgHQeQCkrYXASBhzJ4y8peU3jhXnwgdXw4Gf4Ozfw1kPgghJ2Ue4fvYa8o6U0zUigMvLPuPO8tlW1U5YN6d8PKcwBl4fSxm+9DvwCPdM6sMD53lAe4MXanPVO0p5nLBuMPUluHstnHaBVYd9/jNw/xbn3Skc3BFmzIfBv7EalD+9DSrLSOgUwid3jqVXdDDJOcWcG7DDmvTGkxI+WFVNw6/HP3sTN8QXMPenfZRWVLk7KlWHJn2lmiKqN1wxG+5ZD2PvAf8Q5+7fxx8ue9Mq6W/+EN7+FRTn0ik0gA9vG8O9E3qScHQz9Jrg3OM6y+DfgE8gtwb9SF5xOfM3Zrg7IlWHJn2lPI2IdZfw5bMhcyP85xzI2UWIvw8PDCjCVlFiNeJ6osAISJxG5/1fMLSznVkrUvDkKmRvpElfKU+VOA2uX2jNY/Cf82Dv95DyIyDWDGaeytGg+0SP7ezMKmLV3jx3R6RqabWkLyITRWSZiLwhIhNb67hKtWmxI6wB1cJj4b3LrYnsuw2xukh6qtgR0DmRYTmfERXsp903PUyjkr6IzBKRbBHZWmf5hSKyS0SSROTRU+zGAEeAACCteeEq5YUiesCNX0PCOdYNY55atVPD0aBry9rEAwOL+W5nNim5xafeTrWKxpb05wAX1l4gInbgNeAiYABwlYgMEJFBIvJFnUcnYJkx5iLgEeBPzvsISnmBgDCY/j5cNhPOvN/d0Zza4CvBN4hp5ht8bMLbK1PdHZFyaNRA4caYpSISV2fxKCDJGJMMICIfAJcYY54FLj7J7g4B/s2IVSnvZveB03/j7igaJyAcEqcRuPVTLk+8mrmr97F+3yG6RwYS2yGI7h0cz5GBxEQEnTgyqRerqKomv7iczmGumf6yiTNRHCcGOFDrfRowuqGVRWQacAEQAbx6kvVuBW4F6NHjFLewK6U81/Ab4ef3eDx2C35BZ5KSV8LOrCK+3ZFNeWX1cat2DPEntkMg3SODrOcO1nNsh0BiOgTi79O+LwrV1YY1qfks2JTBV1uzKDhawarHJtEp1PmJvyVJv0mMMZ8AnzRivZnATLDuyHV1XEopF4kZBl0GEbr1Pf50+23HxgmqrjbkHinjwKES0g4d5UB+CQfyj5J2uIRNBw7z1ZZMKquP/9fvGOJP1/CAY48u4YF0iwigS1gAXcMD6Rzu3+YuDMYYNqUVsGBTBl9szuBgYRmBvnbiOwaTX1xOQUmFxyX9dKB7rfexjmUtVmuOXGfsTinlDiLWdJQLH4D0DRBrTeloswmdwgLoFBbA8J4nblZVbcgqLCUtv4QDh46SdqiErIJSMgpKSc0rZlVyHkWllSds1zHEjy7h1kXAujAE0C08kE5h/nQOC6BzWECjp750pV1ZRczflM6CTZnszy/Bz25jQt9opp7ejXP7d2LJrhzunLuBKhfd39CSM7AW6CMi8VjJfjrwW2cEZYxZACwYMWKEm2eHUEq1yKArrHkD1s86lvRPxW4TYiICiYkIbLC++EhZJVkFR8ksKLUeh0vJKrTe788rYXVyHoX1XBiC/ezWBSe05kLgT6fQgOMuDJ1C/Ql28sUhNbeYBZsyWLA5g90Hj2ATGJfQkbsnJXDBwC6EB/4yAU/NxDmVVW5M+iLyPjAR6CgiacCTxpi3RORuYBFgB2YZY7Y5Iygt6SvVTgSEwaDLYeNcyNpizWbWoafjOc56jugBvk2rxgjx9yGhUygJnUIbXKe4rJLMglKyC0vJLirjYGEpBwvLOFhUSk5hGZvSDnOwsJTSiuoTtg3x96FTmD+RQX6EBPgQGuBLiL8PoQE+hPj7HHttvfd1rONDqL8PIQE+BPraySosZeHmTOZvymBzmjVM/Mi4Dvz5koFcNKgrHUPq789ir6kGc2dJ3xhzVQPLvwS+dGpEaElfqXZl0h+sYaPzkiB7B+xeBFVlx68T0qXWxaDWBaFDTwiLadbE78H+PiR0CiGhU8PjIxljKCytJKfIcUFwXBiyi0o5WFjK4ZIK8ovL2Z9XQlFZJUWlFfVeJOqyCdQ0SwyKCeeJyf2ZMrgr3SICT7mt3S6AOaFdw1k8cmjlWiX9W/bs2ePucJRSzlRdbd1kdngfHNp34nNhGphaidXmA6HdIDzGujM5zPFc+xEQ0WoTylRUVVNcVklRaSVHjj1XHHt/xPEc6GfnosSuxHcMPvVOi3MhdRmkLKNk9xLKC7JI+e1yhvbt1awYTza0svtbNeqhJX2l2jGbzZpkJqzrcXMQH1NVYc0kVvtiUJAGBenWxDWFGVBdcfw2vsGOC0DNhSH2l/dhMdYw1H6NSL6N4Gu3ERHkR0RQC+ZmLsmHfSsgZZmV7LO3//I5ArsQIcXYi7OB5iX9k/HIpK91+kp5MbsvRMZbj/pUV0NxtnURKDhgXRAKa16nQ9ZW6+d1BYRb3xjCujkuOo6LwbFl3awxjVzxjaG0wJp6M2UZpC61YsSAT6B14Uv8NcSfBd2GYjYvgM9vIL7jqauCmsMjk76W9JVSDbLZILSL9WioR1BlmeNCkAaFmdbrwgwocrw+uNUx7WWd6m2fgOMvBIERYPez5jmw+1tzKNv96lnm71jm98uy4lxIWWqV5DM3WVVWdn/oPgrOfhzixkPMcGvdWoIDrPehvriERyZ9pZRqER9/iOxlPRpSVQFFWb9cCAozfnkUZcL+n6CsACrLrYZnc+oG3BPYfK1RR8c/aA2HHTvq1D2VbI60XH1il1Nn8Mikr9U7SimXs/tCRHfr0RjVVdY3iKqyXy4EVRUnLqssh6pyqw0hdiT4BTUtrmNJ3zVTTXpk0tfqHaWUx7HZHQm8iUm8OccBlyV9nTlLKaU8SUhnGHCpyybK8ciSvlJKea3OA+DKt122e48s6YvIVBGZWVBQ4O5QlFKqXfHIpG+MWWCMuTU8PNzdoSilVLvikUlfKaWUa2jSV0opL6JJXymlvIhHJn1tyFVKKdfwyKSvDblKKeUaHpn0lVJKuYZHTqJSQ0RygH3N3LwjkOvEcJxF42oajatpNK6maa9x9TTGRNf3A49O+i0hIusamjnGnTSuptG4mkbjahpvjEurd5RSyoto0ldKKS/SnpP+THcH0ACNq2k0rqbRuJrG6+Jqt3X6SimlTtSeS/pKKaXqaJdJX0QuFJFdIpIkIo+6+FjdReQHEdkuIttE5D7H8qdEJF1ENjoek2tt85gjtl0icoGr4haRVBHZ4jj+OseySBFZLCJ7HM8dHMtFRF5xHHuziAyrtZ8ZjvX3iMiMFsbUt9Y52SgihSJyv7vOl4jMEpFsEdlaa5nTzpGIDHf8DpIc20oL4vq7iOx0HPtTEYlwLI8TkaO1zt0bpzp+Q5+xmXE57XcnIvEistqx/EMROX7W8KbF9WGtmFJFZGNrni9pODe49+/LGNOuHoAd2Av0AvyATcAAFx6vKzDM8ToU2A0MAJ4CHqxn/QGOmPyBeEesdlfEDaQCHess+xvwqOP1o8BzjteTga8AAc4AVjuWRwLJjucOjtcdnPi7ygJ6uut8AWcBw4CtrjhHwBrHuuLY9qIWxHU+4ON4/VytuOJqr1dnP/Uev6HP2My4nPa7A+YB0x2v3wDuaG5cdX7+PPDH1jxfNJwb3Pr31R5L+qOAJGNMsjGmHPgAuMRVBzPGZBpjNjheFwE7gJiTbHIJ8IExpswYkwIkOWJurbgvAWqm5XkbuLTW8neM5ScgQkS6AhcAi40x+caYQ8Bi4EInxXIOsNcYc7Ib8Fx6vowxS4H8eo7Z4nPk+FmYMeYnY/2HvlNrX02OyxjzjTGm0vH2JyD2ZPs4xfEb+oxNjuskmvS7c5RSJwEfOzMux36vBN4/2T6cfb5Okhvc+vfVHpN+DHCg1vs0Tp6EnUZE4oChwGrHorsdX9Nm1fo62FB8rojbAN+IyHoRudWxrLMxJtPxOgvo7Ia4akzn+H9Ed5+vGs46RzGO166I8Uaskl2NeBH5WUR+FJHxteJt6PgNfcbmcsbvLgo4XOvC5qzzNR44aIzZU2tZq56vOrnBrX9f7THpu4WIhAD/A+43xhQCrwO9gSFAJtbXy9Z2pjFmGHARcJeInFX7h47SgVu6bznqan8FfORY5Ann6wTuPEcNEZEngEpgrmNRJtDDGDMUeAD4r4iENXZ/TviMHvm7q+Uqji9ctOr5qic3NHtfztAek3460L3W+1jHMpcREV+sX+pcY8wnAMaYg8aYKmNMNfBvrK+0J4vP6XEbY9Idz9nAp44YDjq+FtZ8nc1u7bgcLgI2GGMOOmJ0+/mqxVnnKJ3jq2BaHKOIXA9cDFztSBg4qk/yHK/XY9WXn3aK4zf0GZvMib+7PKwqDZ964m0Wx76mAR/WirfVzld9ueEk+2qdv69TVfq3tQfgg9XQEc8vjUQDXXg8wapLe6nO8q61Xv8Oq24TYCDHN24lYzVsOTVuIBgIrfV6JVZd/N85vhHpb47XUzi+EWmN+aURKQWrAamD43WkE87bB8ANnnC+qNOw58xzxIkNbZNbENeFwHYgus560YDd8boX1j/+SY/f0GdsZlxO+91hffOr3ZB7Z3PjqnXOfnTH+aLh3ODWvy+XJEJ3P7BawXdjXcGfcPGxzsT6erYZ2Oh4TAbeBbY4ls+v84/xhCO2XdRqbXdm3I4/5k2Ox7aa/WHVm34H7AG+rfXHI8BrjmNvAUbU2teNWI1wSdRK1C2ILRirVBdea5lbzhfW1/5MoAKrTvQmZ54jYASw1bHNqzhuiGxmXElYdbs1f2dvONb9teN3vBHYAEw91fEb+ozNjMtpvzvH3+0ax2f9CPBvblyO5XOA2+us2yrni4Zzg1v/vvSOXKWU8iLtsU5fKaVUAzTpK6WUF9Gkr5RSXkSTvlJKeRFN+kop5UU06SuvJCLPisjZInKpiDzWwDp9RWSJYyTGHSIy07F8iNQaSVKptkSTvvJWo7EGLZsALG1gnVeAF40xQ4wx/YF/OpYPwepvrVSbo0lfeRWxxqTfDIwEVgE3A6+LyB/rWb0rtQa0MsZscYwZ9DTwG8c3gN+ISLBjoLE1jkG8LnEc63oR+dzxbWGPiDzpWB4sIgtFZJOIbBWR37j6cytVw+fUqyjVfhhjHhKRecB1WINtLTHGjGtg9ReB70VkJfANMNsYc9hxgRhhjLkbQET+D/jeGHOjWBObrBGRbx37GAUkAiXAWhFZiDV/QIYxZopj+3CXfFil6qElfeWNhmENT9EPa4zzehljZgP9sYYDmAj8JCL+9ax6PvCoWDMzLQECgB6Ony02xuQZY44Cn2Ddmr8FOE9EnhOR8caYAmd8KKUaQ0v6ymuIyBCssVhigVwgyFosG4ExjsR8HGNMBjALmCXWVHyJ9e0a+LUxZled443mxGFzjTFmt2MqvMnAMyLynTHm6ZZ8NqUaS0v6ymsYYzYaY4bwy7R13wMXOBpqT0j4Ys3j6ut43QVroKx0oAhr+rsai4B7auYnFZGhtX52nlhzogZizWq0QkS6ASXGmPewRlwchlKtRJO+8ioiEg0cMtbY7/2MMdtPsvr5wFYR2YSV2B8yxmQBPwADahpygT8DvsBmEdnmeF9jDdZ46puB/xlj1gGDsOr9NwJPAs849UMqdRI6yqZSLuKY8ORYg69SnkBL+kop5UW0pK+UUl5ES/pKKeVFNOkrpZQX0aSvlFJeRJO+Ukp5EU36SinlRTTpK6WUF/n/AFN1iKMn/L8AAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 432x288 with 1 Axes>"
-      ]
-     },
-     "metadata": {
-      "needs_background": "light"
-     },
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAADyCAYAAAC1fYsJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABj5UlEQVR4nO29d3hc5Zn+/zkzo957sSQ3uTdJtoyBgCmhxOACoW/oJIENhIQUwm6Wddi0b0jYJUsu8ttQ08BgE2zAmJBQEqq7imVJltXrqNfp8/7+GJ/jmdGMpmhGxT73dcFljc6c887ovPd56v1IQghUqFChYiJopnsBKlSomPlQiUKFChU+oRKFChUqfEIlChUqVPiEShQqVKjwCZUoVKhQ4RM6H79Xc6cqVIQf0nQvwBdUi0KFChU+oRKFChUqfEIlChUqVPiEShQqVKjwCZUoVKhQ4RMqUahQocInVKJQoUKFT6hEoUKFCp9QiUKFChU+oRKFChUqfEIlChUqVPiEShQqVKjwCZUoVKhQ4RMqUahQocInVKJQoUKFT6hEMQ0QQmA2m7FarajjElTMBvgSrlERYtjtdsxmM0ajUXlNq9USERGBTqdDq9UiSTNex0TFWQbJxxNNfdyFCEIIrFYrVqsVSZKwWCzK60II7Ha7QhAmk4mEhAQiIyNV4jg7MOP/wKpFMQWQXQ1nMpAhSRKSJKHRaJRjT548ybx584iNjQVUi0PF9EMlijDDarXS2tqKzWZjzpw5SJKkWBGeNrxMHFqtFq1Wq1gbBoNBOV6n0yn/qcShYiqgEkWY4Oxq2O12xeUIFJ4sDpvNhtVqVY7R6XSKxaHRaFTiUBFyqEQRBtjtdiwWi+JqyFaEv5joePl8MtyJQ5IkF4tDJQ4VoYBKFCGEvGnlQKVsBXjb+N7cj0DgiTisVquyBtka0el0REZGqsShIiioRBEiCCGwWCzYbLZxm9edKHxZGYFaIO7vdSeO1tZWAHJyclSLQ0VQUIkiBJBrI2QLwVNmY7oKq5zXIwdHLRaLi8Uhxzi0Wq1KHCo8QiWKScC9NkJ2NdwRyhjFZCFnVGR4Ig45MKrT6TwSn4qzDypRBAn32oiJNlMwGz/UROFtfZ6Iw2w2YzKZAEecJSIiQrE4VOI4O6ESRRCQA5beXA13BGNRTBcmIg7nwKizq6LizIf6Vw4Aspl+9OhRTCaT3/78dLsekzmXTBxy4BPAbDZz4MABBgYGGBoaYmxsTEkHqzgzoVoUfsK5NkIOXPqL6QxmhhLOQVGj0ahYHmazGbPZDKBaHGcoVKLwAffaCNn8DidRzBZicS43h9MNbu7E4dynohLH7IRKFBPAW22EJEkBmdmzZeMHCm8NbjJk4jCZTOOCoypxzC6oROEFE9VGhNtCOFOIxRdxCCFc3BQ5Hati5kElCjc4uxreaiM0Go1qUQQBT8Rht9sVEZ/29nby8vKIjIxUO2NnGFSicIK/tRGTtRCEEHR0dBAREUFycjI6nW7C489UuH/HnZ2d5ObmqupfMxAqUZyCrzJsZ0wmmGkymSgvL1dEaRobG9FoNCQnJ5OamkpiYuLkPsgsh3sNh7sWh0oc04Oznij8LcN2RrDBzL6+Po4fP87ixYtJTk5WLBez2czAwACdnZ3U1tYq3Z8RERHEx8eftZvBkxaHShzTg7OaKOx2O3q9HiEEKSkpft9kwbgGw8PDnDhxgrVr1xIdHa2kWwEiIyPJzMwkMzMTgJqaGjQaDc3NzYyMjBAbG0tKSgqpqanExMQELYAz2+EPcajqX+HBWUkUzgHL4eFhhBCkpqb6/f5Agplms5nKykqEEJSWlvplsURERJCYmEh6ejpCCMbGxujv76eurg6DwUBCQoJCHFFRUT7Pd6bGO7ypf8mBaBmRkZFERUWpnbGTwFlHFO6uhlardXm6+wN/LYr+/n6qqqqYN28enZ2dQdUMSJJEXFwccXFx5OXlIYRgeHiYvr4+qqqqsFgsJCUlkZqaSnJyMhEREQFf40yBJ+Jobm4mIiKCzMxMl5Z6VYsjMJxVROFJoi7QVCf4DmYKIWhoaKC7u5uSkhI0Gg0dHR1+n9+XqE1iYiKJiYnMmzcPm83G0NAQfX19NDc3I4RQAqNJSUkuwcGzDfL3KLsiqmxg8DgriGKi2ohAMxgwcTDTbDZTUVFBXFyc4mqEszdEq9WSkpJCSkoK4FD9HhgYoKenh5MnTypq3vHx8djt9rOuEtL5M/sjG6gSh2ec8UQxkUQdBJ7BkN/jaSPLrsaiRYuUwOREx4cDOp2O9PR00tPTAQdx1dXVMTAwwMGDB4mKiiI1NZWUlBTi4uLO+I0wETl6Ig5V/cszzmii8Kc2IljXw/k9QggaGxvR6/WUlJQQExPjcvx0lnBHRkYqwc+cnBwMBgN9fX00NjYyOjpKXFycQhzu655KhItIA7GiAlH/OtuI44wkikBqI4IhCueNLGc1YmJivGY1ZlKlZUxMDHPmzGHOnDkIIRgdHaWvr4/a2lpllKFMHJGRkVO2rlAoknvCZNytiUR82tvbycrKIjY29qyQDTzjiCIQiToInihsNhsDAwMcO3aMwsJCsrKyJjzenSgmuu5UEYskScTHxxMfH09BQQF2u13JqLS1tWGz2UhOTiYlJcVjqXkoMROJwh3OxNHX10dWVpaL+pdscZyJWhxnFFHIZqL8R/PnxguWKHp7e+ns7KS4uFgpx57oePeNP1MsDGdoNBqSkpJISkpi/vz52Gw2BgcHFVdFkiQloxLq9Ycr0Bqu89psNiVQDLhYmGazmfvvv58f/vCHLF26NOTXng6cEUQhuxrV1dUkJye7BBJ9IVCisFgsNDc3I0kS69evD8tNGA6LIpintVarJTU1VSlGs1gsDAwMoNfrGRsb48iRI0rhV0JCwqQsgtlgUbif19ktcS4pB4fFMZ0xn1Bj1hOFc21EOGoinDE4OEhlZSVpaWkuGpK+cKb4rREREWRkZJCens7Q0BDLly+nr6+P1tZWhoeHiYmJUeIbsbGxAX1uIcSssihg4r/r6Ogo8fHxYbnudGDWEoWn8X1arTYoN8LXe+QKv46ODoqLixkZGWFwcDDotTuf19PNFmqLItTWibzuqKgocnJyyMnJQQihZFTq6+sZGxsjPj5eIY7o6OgJzynHlEKNcBGFr+9U/vxnCmYlUXirjQhFqtMdFouFyspKoqKiKC0tRavVMjo6qipOu0GSJGJjY4mNjVVKzUdGRujr66O6uhqz2UxSUpJSHOZeaj7bXA9fsFgsU5o1CjdmHVFMVBuh1Wqx2WwBnW8iohgcHOTYsWPMnz+fnJwcl/dM5iktC9dYLBZSU1PHBUNnUjrVE/zZ1JIkkZCQQEJCAnPnzsVutzM4OEh/fz8tLS1KqbmcUQnnkz9ccaSzCbOGKPypjdBoNAE3eHkiCiEELS0ttLW1sWbNGuLi4lx+P5mNbLVaqaqqAiA2Npa6ujqMRiOJiYmKmT7TEczTX6PReCw1l10V+bwDAwMkJibO6NSir799uKyj6cSsIAp/ayNC4XpYrVYqKyvR6XSsX7/eY1NVMNcBR4CrvLyc/Px8srOzsVqtSv2C3NjV0tKC2WwmJiaG2NhYkpKSZtymCYW1415q3tvbS0tLiyLeExUVpRDLTBPvkVOjE+FMI4sZTxSBjO8LNpgpY2hoiMrKSubNm0dubu6E7wl0s1gsFsrKyli5ciWJiYlKByOgSOElJycD0NraysjICHq9nhMnThAZGamkKWdKf0ao16DVaomNjWXx4sUAGAwG+vv7FfGeuLg4hTiCFe8JFXwRhdVqPeO6dmcsUQQjUafRaAKOUcjXamlpobW1ldWrV/uMVgdiUdjtdk6cOIHFYuG8885Tgni+CC8uLo78/HwAjEajS3+GnE3wV7gm1AjH09L9nDExMcTExJCbm6uI9/T19SmumnNGZaq/A5vNNuH9KPfQnEmYkURhs9kYGRkhMjIyoPr5YFwCq9WKwWBgcHDQq6vhDn8tCpPJRFlZGWlpacTExPgtKuN+/ujoaHJzc5VNI2cTqqqqsFqtLsI14SyzlhEOovDV5SmL9+Tn52O32z1+B3JgNNziPe7FVu4402ooYIYRhVwbYTKZOHLkCBs2bAjohgyUKIaHh6moqECn07Fy5Uq/3+cPUcgt50uXLiUtLY2uri6/z+/r2s7ZBPcyazloKFdLTkexUTAIhHw0Gs048R45oyKL98ikEY7skS/XQ7Uowgjn2gg5/RjozehvelQIQVtbG83NzaxevZry8vKArjMRIQkhaGpqoqurSxHSDRSBCtc4l1mbzWb6+/tpb29neHiY6OhoJEkiKSkpZJZAODbfZNbm/h1YrVb6+/vp6enBYDAopeYpKSkhIU9fRDEyMqJaFOFAIDM1JoI/FoVzenL9+vVBmereNrKcMYmMjPRbSDfUiIyMJCsri6ysLKVasq6ujq6uLjo7O0PSRj7Vrkeg0Ol0ZGRkkJyczNjYGMuXLx9HnjJxBBMc9hWjkIOvZxKmlSj8Gd8XCHwRhexqFBQUkJeXF9LrjIyMUF5e7jNj4g9CVXAlV0smJSURHR1NRkaGSxu53W4PSl9zKoKZoYBMPlFRUWRnZ5Odna2QZ39/v0tw2Dmj4gu+LIozrXwbppEoAtWN8AcTuR5tbW00NTWxatUqEhISPK7H3zW4b+SOjg4aGhq8njuYa4QD7m3k7vqaOp1OMeEnql0Ih+sRjspMT+d0LjWXxXtGRkbo7+9XxHsSExMV4vBkdfkTzFQtihAgVK6GOzw96W02G1VVVdjtdq+uhrzxAyUKu91OTU0NRqMxaDdmovOHG+5FT0ajcVztgkwc7rGW2WRRTATn4LC/4j02m23Cv7Wa9ZgkAq2NCPTmcSeKkZERKioqyMvLIy8vz2dFp79PNI1Gg9Vq5cCBA2RmZrJ06VK/eh+m26LwhejoaJduUFkmT27qkjeMnLYOJcLRkxGMleJJvGdgYEBxVeQHW0pKitfzj46OBjRQajZgyojC00yNiSC7EYE8pZ3P2d7eTmNjo093AAJPq/b19TE6Osq6dev8viECsRJmQlOYu0yecwqyp6cHs9lMfX29Mlh5spvcbreHvAbEl4vgD7RaLWlpaaSlpQGOCtvq6mqGhoY4ePAgERERSnBYFu8ZHR1ViuX8hSRJVwJPAlrgGSHEz91+XwC8CCSfOuYHQoi9k/pwASDsROGuG+GvqxFMObZ8vcrKSqxWq9/ugL9EIQ/26enpITY2NqCnxkzY/JOBcwoyMzOT5uZm4uPjld4MOZMgd8MGanGEw9rylZ0IBhEREURHR5Oenk5KSorirsniPfv376e8vFwJnPrzmSRJ0gK/AS4DWoEDkiTtEUJUOR32Q+AVIcTTkiQtB/YC80L64SZAWInC3dUI5EYIpmV8dHSUsbEx8vPzyc/PD2lFp8VioaKigtjYWNatW8dnn30W0NrciWKitc0UKTxvEEKg1WqVwcrOmYT6+vpx81H9ScNOVTAzFHDOeri7a0lJSXz88ce89NJLPP300zz//POsXbvW1ynXA3VCiHoASZJeBrYCzkQhgMRT/04C2kP5mXwhbEQhByz379/P+vXrg2pLDoQo5MxDdHQ0BQUFAV3Ll8qV3Cy2YMECsrOzAzq38zUCcT1CiXApXMnwlEmQu2ErKyux2WxKFiE5OdmjOzDTazOc4S09KkkSS5cuJT09ne3bt1NSUuLvdz8HaHH6uRU4x+2Y7cBfJUl6AIgDvhjU4oNEyInCvTZCVqEKFP5aFDabjerqaiwWC+vXr2f//v0BX2siIRq5gtOTLkUgcCYKu91OVVUVg4ODijnv3k4+k90UX5targR1T8PK2hM6nW6cKO9sIwp/Cq5C3EF6M/CCEOJXkiSdC/xBkqSVQogpkVoLKVF4k6gLBv4QhazvkJubS0FBgXK9yWZLwHEzHD9+HJvNRmlp6aQDbfJmMBqNlJWVkZWVxbx58xgcHFTayaOjo0lNTVUG6s5UBLo29zSsyWRyEeWNi4vDYrGEvPZgqhS43TE6OuozgO6GNsA5+pl36jVn3A1cCSCE+FSSpGggHdAHcqFgETKiEEJgMplCVhvhK27Q2dnJyZMnWblyJUlJScrrwTyd3K81NjamEFAgsY6JIEkSAwMDnDx5kmXLlpGcnIzZbCYjI4OMjAwXcdrW1lYMBgN2u12JqE9FV2ggmMx34i7KOzo6Sk1NDa2trTQ3N7t0w06mE3SqXQ8ZY2NjgRLFAWCRJEnzcRDETcAtbsc0A5cCL0iStAyIBroDuchkELK7TyYHTzdQKBu87HY71dXVmEwm1q9fP+5GkrMlgdwgzkTR3d1NbW0tK1asUIRkJguZBOrr65VGMXcSdPbz4+Pj6erqIjMzk76+PpqampSu0LS0tEnP0JgsQukmyGnYuLg4cnNziY+PV7phm5ubAZT4RqBqX+FIucrnDaUehRDCKknS/cA7OFKfzwkhjkmS9BhwUAixB/gO8DtJkr6NI7B5h5hCszOk36InK0B+LVB/zRNRyE/67Oxsli1b5vFmlYOggdwg8nvq6uro7++ntLQ0ZArKNpuNY8eOYbVaKSkp8bubVC7qkTUmzWbzOHPdW9VkuBHOXg93bU2LxUJ/f7/insnT2P1Jw4ZTsHei6wZ6/506514cKU/n1x51+ncVcH5gKw0dwm7Pyht+skTR1dVFXV2dzyd9MOI1Qgjq6upIT09n3bp1IdsEBoOBsrIycnNzA5pb4SlDEhkZ6dLYJFdNHj9+HKvVqjR3ecsqhBJT2esRERGhpGGBcbNDnLth3ZWuwkUUE/0dZ3JsaTIIO1HodDqsVmvAT2itVqtUctbU1GAwGDy6Gp7eF0hadXBwkPb2dnJzcxW9xlCgt7eX6upqli9fTkpKCgMDAyG7iTxVTbpnFdLS0hSNzXBguno93Kexy30ZstKVM2FO10yPUPYvzRSElCg8fTnBFE7J7xsZGWH//v1kZWX51U8BgVVZtra20traSm5ubqDBJ69BUyEEjY2N6PV6F+EaT1aCt88TaMGVe5mxu8amJEmKeG8oXKqZ0mYuSdI4pStnwjSbzVitVnQ6XUjjOv7I9Z9pmDLXI1AMDQ3R1tZGcXFxQLMu/CEKOW4gSY5Bw62trUGPInQ28202G5WVlURERIwTrpnKEm53jc2amhrMZjMVFRWKTJyn2g1/MVNrHtwJ89ixY0RFRSlxHbnsPjU1NegBwr5cSLPZHHbNzunAlLgegRCF3W6ntraW/v5+srKyAh6I46ui03m2hixeE0xcw33jj42NUVZW5nLeiY4P5NyTgSRJREdHExcXR0ZGhjKRvKury6V2w9PEsqlEOMhHkiQyMzOJi4tzUfJ21p2Q4xv+bu6zUYsCZpjrYTAYKC8vV1q329sDL2efqJmsq6uLkydPsmLFCpfai8lOGOvp6aGmpmZcTYczprMpzPm68kRy99qNEydOYDKZlBqGiWo3ZqpF4Q7nCkpJGq/k7Tx0yV9Lyx9h3TNNiwKmyPVwHnbjDXL6Sw7+DQ8PB9U96sk6kGdrjIyMsG7dunF+erAWhd1up76+nt7eXtatWzfhfAlPROFtw00VqTjXbuTl5SnzQZ1rN2Rrw9nHnykxCl+YiHzchy7Jgry+hi6djQrcMANiFM6b2Ll+IdhhPu4Whclkory8nNTUVEpKSkK6MY8dO0ZsbCxr1671S0nJ32DmdMG9hsFb7YbcDRxKTLeVIgvyZmRkAHjU1ZQ7Yc+24T8wBa6HTqfDbDZ7PF7ue0hPTx+3iYMNgjoTjDxbY8mSJUqfgbf3BGJRjI6OMjAwwMKFC5k/f75f75nuNvNg4Kl2o7e3l46ODmw2G0ajMWS1G9NNFO5wn1QmDxxqbW3FaDRSW1vrcehSMK6HL9GaU8fcgKODVABlQgj3Eu+wYtpcD7lUetmyZR4FYCZLFI2NjXR2dlJSUuIzwh0IUcimaVJSklIE5A+ms808FHCu3YiIiMBqtRIXF0dvby/19fWK0tNk5qPOJKJwhiSd1tVMSEigp6eH9PT0cUOX4uLiGBwcDMiiOHWPTyhaI0nSIuAR4HwhRL8kSf7feN4/kxbQCSFM/hw/5a6H3W6nrq6OoaGhCf36YIlCro9ITk5m/fr1fs8s9UUUQghOnjyplHhXV1cHZIUEaiXMBIvCG2ThmolqN+SMQmpq6rSmC0NNPjabzYUU4fTQpb/97W/87Gc/Iy4ujoKCArZs2cKcOXMmPN8pWQRfojVfBX4jhOgHEEIE3DEqSdI8YCFgxNF4tg04CbwhSZLkq29kStOjRqOR8vJy0tLSWLt2bcjN75GRERoaGkhMTGTFihV+v88XUcjqVnFxcUo8orbPwksn6mketNA/ZiYlNhIQ9I9ZSYnVUZgRz7aiHIrzkz1+HoPBQEdHh8chNDPF9fAGT0K47rUbckahtbVVySikpaWFRF9zOuEpmCkPXbrlllswmUx0dXUpaX5fRNHW1ga+RWsWA0iS9DEO92S7EGKfP+uVJClCCGHB0Y26DqgBNDiI4qfyYThcGq+YkvSo1WpVUojyLM5gzjURZIWrefPmeY2JeMNERCEP9pk/fz45OTnsONjKi5+1cLJ71O3IMZefDjYNsuNgGwsyYrl9QwHrUk5vfrm8Oysri8bGRsbGxkhMTCQtLS3gupGZCHfhGrmxy1lfMzU1VRnePJvgS7RmbGyMhQsXctddd4XysjpgEXARDq2Kf0iStEoIMTDRm05ZCnLevw6H5mbdqd/t57Scns+nUtgtCo1Gw+DgICaTyWcKMRg4z9YoLS1lcHAQo9EY8Bo9PcHluotVq1ZRN2Dju88d4GDToN/nFcDJ7jEefaOalZlR3HtuDmZzMx0dHYpFJadZh4aG6O3tpampCXCk64aHhyccxDNdCDTw6NzYJddu9Pb2KoVPSUlJirbqTNPdcIev1vVAsx6nLA5fojWtwOenNn2DJEm1OIjjwETnFkIISZKuAY4JIV6RHNABdmA3YJOP87XOsP5VTCYTlZWV2O32kHZlypCzJs6zNYKJbcibVYbcTTo0NERpaSmP7K7hjYrOSa21Um/i/t2NLEvX8R9b1hAVFaVYPu45/aGhIaqrq5VBPAkJCUqTV7D+fii/+8lkKJxrN+TCp8HBQbq7uzl69KjX2o2ZApvNNuHDLtCsR2lpKfgWrXkdhxTe85IkpeNwRer9vEQucJkkSe8CbzpZGAEVDoXN9ZDN60WLFtHQ0BDyP7h8fndXJpjiKef3WCwWysvLSUxMRMpYyHXPHKK6c2TC989JjiInKQY5RhGhhZrOUY/23PEeK7c8d4h7zp/LNzd6FgGOiooiKiqKFStWKB2Svb29tLa2Aihm+3RtpFDGT+SMQVRUFOvWrfNYuyGTZKit0WAQ6rmjp6wTX6I17wCXS5JUhcMK+J4Qotef8wshfiNJ0teALcBcSZLeEkKc8HuB8joDfYMfC1OyA2vXriUqKoq6urpJnc95M8jdmd3d3S7dmTImQxTyEOOFCxfy+7Ihnv34oMfNLgHzknSsyUvkpnMXKAFLZxxpGeCX79ZxqGnA4zme+biJI819fOuShRTlJXo44tS1nDokZX/feSPFx8crG8lbZ2ioA6PhqHmQ4V67IdcvOLeRp6Wl+RyqHExVrz/wFaMIpuDKD9EaATx06r+AIYT4P0mSdgC/A66VJOkp4K++YhzOCLm47uHDh0lMTPSrWtEX3KeFWSwWKisriY6OZt26dR7PH8zgII1Gg8FgoKKigtWrV/P0Jx0883GTx2OXZsex/eplJJh6SExMpM0E/98/Glg/3xGE3N/QT3JsBANjFu4sTuGKbAN7WyM50jY87lyHWoa58w9lPH/rGheymCjrERERQVZWFllZWcpG6u3tVVw82dpITEwMq7URalfG2zXk2oW5c+dis9mUSWV1dXVey6zlc06HXuZM6/WQJOk2HDGQEeCvwG3Aj4EiSZL+VwjR4c95Qu56LF++PGSRbGeikJ/2vmZrBFr6bbfbOXnyJCaTiY0bN/I/7zd6JYnNq7L5l3Py2N/Qj2V0mKahQfbVDGC1C3Qax01qsQkEDqtDK8G1JTlcU5KE1dpIRdf4IKvVLnj+0xaevN7/dK4M5400b948JbvQ3t5OdXW1Yrb702sTCEJtUfh7Pq1W66LmLTe0NTQ0KJkj5/qG6VDglmNKMwjLccQpOoFa4F+BY8ADwB8lSdoqhJjYtyYMrkdsbGzITF2ZKOQ5oqtXr/bJ1oG4HmazmfLycpKTk4mJifFKEuvmJrFldQ5VHcPc9vwhhQycYbE5XpFfF4BVwCuHOnjlUIdCHKlxEXSPuHaqvl/by6uHO7i+JAcIvo7CPbsgl1z39PTQ29urzBEJVodCxnQRhTuc1a6cp5DL+iLyPJFQ1m6EQYE73Ph3IYSnJ+evJUn6Mn4GNUNOFN5u8mBuBo1GQ21tLUDI54gODQ1RUVHBokWLyMzM5Kl/tvB243iSuOf8uXxxWQZ3vHAYk9XuNWYRoR1vUTgfKwCbgJ4RCzqNhNUuXH73X2+foKXfwEOXLvC5dn/gXHJts9mIi4tDo9EoJegxMTGKmzLV4rzuCAXxuE8hHxoaora2ls7OTmpqaoiJiVFiOZOxeP0hiunU9XCHEMImSZLm9I+OzSk5vvCrhRBj3t99GlOStA5GiVvOtefm5rJkyRK/byR/0qPt7e00NTVRVFREXFwcj//1BG83jn+PTBJPvV+P2TaeJDSATitxbXEuF8+Lpam5iYHITLJT4qnqGOa1I+1YbQI7p4lDAHa7YGFGHPXdoy4WyPOfObIa39xYEPIApGy2yzoUsoiLPGVN1mJITk72+fQNtUURDi0KjUZDbGwsS5cu9Shak5SURFpa2rimLl/wFcwMxQT1UEN4mCZ2ijDGB868YEqIQi7j9vcLlKs4U1JSyMzMDOimnOhY9+IsnU7HjoOtPOvB3di8KpsRk9XF1dBIoNM4iCEnysqgycZla+aRah/g45o2RmJzyU6IZmDMwvKcBByuISzPSeBgXRdv1wxgszuIo6FnFK2bZQHwwmet5CVHUhjGrKe7iIt7kNBX5eRMcT0mgjP5eBKtkbU15aYuf2s3JiK1mVx2P1mExfVwh1zG7UvY1Tm1um7dOpqamoJqDPMEk8mktLTLxVlHWgb40Zs14yyFzauyefe43sXV0ADnLUjl/osd6dB3Dp3gcOsw7x+uoXHQxgdNBiy2Rhe3QyYWyOWKpSlsnBfNX2pNfFLfh10AQpCbFEX74OkGPgH89J16HlkfTUlIPrlvuAcJ3Z++ckpSbicP9YaY6knmzsQAjnujv7+flpYWRkZGFO0JT7UbcjWtN/j6/WzFlFgU/rgDsvhrfHy8kloNJtXpCYODg1RWVo7Tpfjluyewud30m1dl0z9mdnE1JCBSp+H+ix3xg/984zi7DrdjOWUNeIpHANgFmG2CHQfb2HUELluYwOXL53CwaQCz1Y4d6Bw0odVI2JwsC5uA5ypNLFk6NGGNRbjgrHpls9kYHBx0aSe32+3Ex8eHzBKYbi2KqKioCWs3nCXyJoIQ4oy1KqbU9fAGeSMXFhaSlZWlvB6sypUzWltbaWlpobi42CXItONg67i+jXVzk1wsCWdXY1uRIyPhKajpfmvIxOEcl7DYYG/tMO/V1/KVc/J597ie5j4DdkArBLlJkbQPnm5max8VHmssgkWwG1Gr1bo8fQ0GA8ePH6e9vZ2WlhbF109JSQnaNw9kOFIg5wzGSvFVu2EwGGhpafE4qcxsNgdVPeqPcM2p474M7ARKhRAHA77QJDBlroe3DS9vZDmw6O/7fEGeUWqxWDxOI995eLxwr9kqFEvC3dU40jLAU+/Xe8x8OBPK8pwEBsYsJMdGKAFN53Sq2WrnhU+bsdlPvyZJoB8eL+47mRoLZ4TyKRcTE0NcXBzZ2dkkJCQo1kZDQ8O4wUP+bv5wFEeFyp1xd8s+++wzNBqNMqnMuXYjmKpMf4RrACRJSgAeBD6f9IcKAlPmergX/dhsNqqqqhBCsH79eo9PI3laWKCw2+0cOHCArKws5s6dO+6G3XGwlcq2IZfXNBJUtA0plkSk1tXVcN7wGgm0GvjignjOXZpHVYcjeOxMEgNjFrYV5bA8J4Gdh9uo6hjBLgQaScIuXOswhPC+mT880cvR1ulxQSaCJI2fE2oymejt7fXYOj9RZmG6XY9AzqnT6cbVbvT29lJRUcEPf/hDdDodn3zyid/pfD+FawD+C/h/wPdC+qH8xLTEKOQZGHl5eeTl5Xm9SbRabcAt4/39/YyNjVFSUuJRJ3PHwVa2v1ntUmWyMCOW+u6xcZYEjHc15N9/pSiV4eFhDrlZDeMDmQ7LQJJgYUoEG5fl8sfPW1zOaROO93lSD7EL2FPRNaOIwhupRUVFKeI17q3zcgAxLS1tXOv8VAczg4V75s65dmPBggXExsbyk5/8hBdeeIF33nmHH/3oRz7P6Y9wjSRJJUC+EOItSZLODKLw5XrIBT8TzcBwfl8gwcyWlhba2tpISEjwOMj4SMsA299wJQmtBPPTYmnqHUMIiHCyJNxdDTmoefnyTPbVdPNWVS9Wu/dApku1poATfRYaP2tmZapEXFwC+1tGlDoLmWTcIYDXjnSwLCteqdycbvhjAbi3zpvNZnp7ez22zs8mi2KiOExERASLFy/m//7v/0J2zVPFUk8Ad4TspEFgyoKZsnLx8PCwiyz/RPA3mGm326mqqsJut1NaWsrhw4c9EswzHzWOq1eVJIn3anrQANcW5/DlEod0mbMlIVsHFy5KBwQ/3lvjsYwbPFsUzsdabIKj3YKIvhG+sDAF/bCZYx0jpy0W6ZQr4nROm3BUboLg+pJcn99HuBHMxo6MjCQnJ4ecnJxxrfOyYM3Q0FDIWud9CcwEA386RwNtCPNDuCYBWAl8cOp7yQb2SJK0ZSoDmlNCFHa7nZaWFubMmeN1toYn+BPMlMVrsrOzKSgomFC8pqHX4PJzTIQGk9WOnJnMSXCQ1/86WRIaYGVuIpkJkfzjRM84gpDLt50DmUNGK9WdwyzNTmDEZKVnxMSHtT2K9SFwpE3fr+1Dq3GQg/0UOQjhCG66s5CjzLuOSEMv5y3O9un3hxOTDY66t853dnai1+sDap33hUAK/EJ1zpGRkYCDmb6Ea4QQg4DiQ0uS9AHw3TMu6zEwMMDJkydJTEyksLAwoHP5Ioq+vj6OHz8+TvLfU7/HjoOtNLjpXBosjmM0EugkSIzWerQkqjuHqWgbTxBaDVxXMkchCID2QaMSs/joZB8ayXGeNZlaxiwStX3W06QAWO2nzydDJgv3/SiA/XpYmT1EY2MjOp2O9PR00tLSpry/IJSuglarJSEhgfnz54esdX4qYhTuCMai8FO4ZtoRtkeSEILmZoc+5NKlS+nt9UuQxwXeYhTyuTs7O/0Sr/EUwFSOxRGcXJZg4q/Hu8dZEiCobB92iVNEaCWuXpFOhs7IILi4Iu7GgF04/jvUeZrwJMa7GO7PaG8P7b3VA1xTPIfi4rmKTHxdXR1Go3FcBWW4EM5eD39b530pXk0HUQSqbiXDl3CN2+sXBXyBECAsRGG1Wjl27Bg6nY7S0lLGxsbo6uoK+DyeYhQ2m41jx46h0WgoLS31Kl4jv89Rpl2Nc0uFwxpwdLlGaB3ByR+/VY3Z6ekuSRJVHUPY7K4xB9nFKGvu47mjw1jtwy6b3FvxFW7HyESgITDxQqtd8F/7TvKfmwpZlROvjMGTJEmZG3ry5EmioqJIT0/3OFxpspjKXg9vrfNVVVXYbDavg4WnI5MyMjJCbu70x5DCgZAThcVi4cCBAxQUFCgzDYItnHJ/n8FgoKysjDlz5pCfn+/1fc4WxTMfNWLz8HTWSHDtKbdh9+FmLC4kgaO022kzn7cglcuXZ1LVMewzmClJUJgkkZOWyIhZcLR1SHE3xlkc/n8dCup7Ddz9pwquLcphaVY8/WNmSvISWDPndJu10Wikv7+fmpoaRkZGMBgMSjpvps3V8HdTO7fOz507d9xgYefW+emyKM7EuaMQBqKIjIykqKjIpevQVwm3NzgThSymK087nwgyURxpGeC9mp5xvxfgqIwUgh/vrcZiO72J3WMDEo5W8ugIjUeCkN2I4oJkQHC4eRC7gNoBwYmBQeW8GgmK85MYMVmp7nKfCRI4rHZ45XCHsoYIrcS2NdksyYylf8zC2vxE1sxx9C/IPRrOG0qe8hVMyfFM6R51Hizs3jo/NDSEEILs7Gy/Wuf9gT/BzJkkgxdKhMX1cFe58jZ/1BfkDd/Q0EB3d7ffc0Hk971+tAO3Lm50Ti5Hd083VpvTE/5U9gFcU6L/ONHD36t7PLoVDusDDjYNjFuHc/zBJuBg8+C4Y0IBOYviThxb12SzKC2KmsY+Ll2Vz4bCfCRJUiTkjh07FlSwMBxEMdmN7N5KLiuX+ds67w/kcYLeEGyMYjZgSoVrAoXVasVgMGAwGLyK6XqCbIn0jIyfv6qRYPOqTOIt/QxICei0/Vjt4lQZ9aljOO1q/LVK7/j9qffLFZQyqUyUKXRvDpsqyMTx6uHTuqlv1DewddUYS7JiGTBYHRbHKsf4P+dgoZyaTEtL87opwhHMDHWqVwhBWlqaEjPw1TrvD2w224RqYMH0eswWTAlRBHNTyWXeOp2O5cuXB/RejUbDnmO9vF8zPtNitQt6e3t5s1NgsTnSlxKnYwWyJeHuargThKedL7sYFy1OZ15aDMc7R0iNi6CqY4T6Hr8Ux8IGs03w6tFOZZ1ajcStpTnERekcpFFYiCRJjI6O0tfXR3l5OYBCGs5l17NB/t89RuGrdV7+nDExMV7X4qsyc3R0dKbpZYYMYSGKYMVhZXR3d1NbW8vKlSs5duxYwO+v7jbxP//sdCvVlhAINBL02mOx2IbHWQQSjpRodeewi6shuxieCEIDrJ4Tj1ajUYKW/6zr5Z91eA14Tjcc9RuC5z9vP+2mrMpysjaSKFqTr4jTymXXcpPXbJDCm+icnlrn+/r6lDSzN5k8NUYxQyCEoL6+nr6+Pr/LvD3hlYp+F5KQgE2FkViEjvcaxpS6CIcFICFJArtwkAmI8a6GhwDnqRVjR+Jo24iLezFTCcITFDfFydrQaSW2rspkSWYcg0Yra/NzWLskUVG5lkcnOBd7TYY4prvXw13N21vrvNVqnW0K3CHDlBLFRDeE1WqloqKC6Ohol+FB8lxQf//oOw628kmT65gCAfy13sw1xelY7aMuGQ67EERoJNbnx3KgZZSKtmHProbTa6cNC8nlGp7+PdsgcBDdzqOOuheZOLaszGRpVhyDRi1JlmjWLl7M0NAQ9fX1GAyGSQnYhGvyWDDnnKh1vq+vD5vNRlZWFqmpqePiKgaDIeBA6b59+/jSl75UgxfRGkmSHgLuAaxAN3CXEKIp4A82SUwZUchVlp5uotHRUcrKypg3b964ghU5EOoPURxpGeCxt8ZrYAJY7IKqjmEXqXy5VMJqFzT2m5VejIlcjfAMqpu5kIljV9npgjmdBNXGVpZmxTNgiGddwRzSExxB0YaGBiIjI118fl8Ih+sRKji3zpeVlZGVlcXw8DDNzc3jWucDzd7YbDa+8Y1vAHwJ76I1R4B1QogxSZLuA34B3BjCj+gXwhajcIeciXAnCjm3v2rVKhITx2suuI8VnAj7G/rHqVrLFZh24RCm0WocRGBzshLsAtqHLMrPnvoswJUz5F6PgtRYGnvHxqVhz2RYBewq0wP6UxZHq8PiyIxl0JTBmuQYwEJtbS1ms9klw+BpI4VzlmkoYbfbFWV4cG2df+CBBxgZGWHHjh1cfvnlflXE7t+/n8LCQk6ePOlVtEYI8b7TWz4DvhLKz+QvptSicFbilhW3BwYGJoxHBFLVOWR0rdWQcOhNLM1JVNSrbE4mgQbIT42huc/gIkvn7GqAd4Jo7hujoWdsVrsak4UniyPilKty9cp81uTGMzg4qNQzeCr2mskWhTPcH3TOrfPvvPMOF1xwAbW1tezbt48XXnjB5/na2trcK4zHida44W7g7eBWPzlMKVHIG95isVBRUUFcXBxr166d8GniL1EcaRng+U9cXTfZrQDHzWu1CyQcwUshHNbGosw4OgaNWE8FID11bAJKF6hKEL4hE8eeSr1TbCOSdQWLmJ8W4aJwnZqaisk0vt5lJmIiy0er1RIREcGjj3rs5Zo0JEn6CrAO2BiWC/jAlLkechn3yMgI5eXlPocNy/BX5erp9+vG9XTIboWzyyEHLy9c7Ki4fK+mB82pKKU3kpCAjPhIekfNKkEEAGdLQwI0GomvrMshIVrH2vy5irWh1+s5duyYonqVlpYWdMYLwjuIxxtRGI3GgAOZc+bMoaXFWQVvnGiNfM0vAv8ObBRCTAurTqlF0d3dTXd3N6tWrfI7jeSPytXfjpzkn/UDLq+lx0rkpiSMczlkK+OEfgSrXSht4BNBAF3D5okPUjEh5P6aF/e3u2RSrl6ZQVxcHPPmzQMcGiOVlZUIIZRAYaCqV9MR8wimKrO0tJQTJ07gTbQGQJKkYuD/A64UQuhDtd5AMSVEIYRgYGAAm81GaWnphPXy7pjI9RBCcOLECV472jlus/eOCYZMw+NcDpkYmvoMHs8pY6rLrs8mOMc19lR08YU5Os41DjBksrM2P5Gi1Y7qyYGBAUX1KiEhgfT0dFJSUnzeP9MR8whG3Uqn0/HUU09x1VVXTSRa8zgQD7x6ivyahRBbQrx832sNx0md2dxisVBeXo5GoyEvLy8gkgDvRGG1WikvL6fVEMGHzaef9s4BSItNsHZuMkdbBrHZBVoJMhMi6RoyT0gCEpAYrWPIaFXJIsyw2OH9FivvtzR6LvbKy2Lt4sXK9C45LSm7KJ7mh4SDKHy5M8H2eWzatAkhxGK3az3q9O8vBnzSMCCsFoVcwbdw4ULMZnPQrebuMQq57mL+/PnsPjigyMkB5CZH0TVkVqorD53q6nS4HNA55NuFEMCgMfBuVxWTw0TFXlevzGBNfj4FBQWKDkVjYyOjo6MuxV46nS5sJeETuTPByODNJoSNKDo6OmhoaGD16tXEx8fT3t4eVHTbPUYhTzqX6y56RlzdtrYBE1qnv6dzQFK1DmYXnF2UNyr1bF2VxdLsWPrHHN2vqxcvRpIkpbS8sbGRiIgIEhISQjKz1hn+NISpRBEgRkdH6ezsdJmWNBmVK4vFghCCpqYm9Hq9oktxpGWAD2q7x73HLsYTQ1ZiFN3DJpfMiEoeswfu/SgRWoktqzK5akUGa+YkEB8fz7x58zCbzXR2djI6Osr+/ftDpiPqj7DumdpiDmEiivj4eIqLi10vNAmVq7GxMSorK5EkyUWX4pmPGl3cDrmiUuAopnKWwe8acrVmJCA5SmLILDxK5amYuZAb2XYe7WJPhX6cste8xESMRiOLFi1SGrxOnjxJdHS0EtuYSFfCE87mzlGYhsrMQGG322lra2PBggUUFBQor+842Mrfq11l7nITo2gfNClNWw4V7dMzRWXIlkS/SXiczuUZcheIipkEd2UvnVbi/BwN15fOZRGQmJio6IQaDAb6+/uV4dUpKSmkpaX5pSMajuE/swnTUpnpLwYHB6mrqyMpKcmFJLw1f3UMmdBqTkvkV7QNOdrIcXUxYiNg9NTsY/+NCZUkZjrkmMYHrTY+6Whg2xqDo3FtzExJfiKrc+PJyspSCv3kYq8TJ04QGxs74RgAXzGKsbGxsCiezxRMeVOYv2hvb6epqYklS5bQ0+NqOXhq/oLTsYmVuYmUy4VWQijKU/JbRgMfkK5ilsGThui2NdlsXpnJypw4hBDKgGFJkhgbG6O/v9+rjqgaowgT3FWudDqdX66HEIKamhoMBgOlpaUYDIZxBJMc61qL4dzt6SyOK/9bnPpfWoyGXsPZ1iiuQo5pvHK4g9fLOrmmKIelWXEMnIpprMyJIyYmRhGwkaXy2traFB3RiIgInzGKM1W0BmaY6yEXZyUlJVFUVOR1jmhVx7DybwlYW5DE0ZYhhwQ/42MS4LhZVJJQYbYJdhxqB05bGtcU5bB5VSarck5rSqSkpCht8aOjozQ1NTEyMsLAwIBHHdFgFLj37dvHgw8+SG1tbR2eRWuigN8Da4Fe4EYhRONkv4NgMGVE4UuJW24WW7hwIVlZWcrr7gVXR1oG2Hn4dN+MAI40D3HneQVUdw7z8ck+NeWpwi/IlsaOQ+385WgH1xTlsCw7nv4xC+sKkliVE4fdbic2NpakpCRSUlLIyMgYpyM6MjIScAm3LFrz7rvvsnDhwuV4Fq25G+gXQhRKknQT8P+YBtEamELXY6KqNllMd/Xq1ePMN3eLwj0lCo44xPOfNHHxknQ0kuSY8jXjoWZRZhLcLY0onYbnbi2iKD+JoaEhOjs7Wb58OVqtVrEo5GKv559/no8//phvfOMbfPnLX+a+++7z2Ukqi9YsWLAAIYTZk2jNqZ+3n/r3TuApSZIkEc72WC+YVnFdIQQNDQ309vZ6Fa9xrsw80jLA+x4mf4FDserv1T0uupZTDa3kiJdoJAm7EOg0ElqNhE4jEanTIATY7HbsNhs6XQRC2DFb7A5hXy0YnEI47mSoYuogAKPVzu7yThal6jh27JhSYWy32xFCYLPZsNvtxMfHc//99/Puu+/y1FNPcfjwYb/6mfwUrZkDtAAIIaySJA0CaYDnTRBGTBtR2Gw2KisriYiIcBHTdYdGo1Esk9ePdowrjtJKp2XtPEhchg3aU/M/EqJ1rM5L4p7zCijKT/J6vEyKg4ODrF692iUwZrPZ6Ovro7u7m8HBQRISEsjIyKDVEMELn7dxsKmfEZNNKR5TMTXYeaSdBZoerrlgjRJ/kO9T+e9nt9v55JNPOHnyJHPmzKGoqGi6lhtWhNX18AQhBCaTiaNHj5KXl0deXp5f53OPTYBDoao4P5H+MQsN3WNhFb6N0WlIjYtgSXaCT1Jwh91up7q6GkmSWLNmzThS1Gq1LjM0h4aG6O7uxtrby52LIvj+efPJyMggOjqaoy2DPPH3k1R1DGOw2FXiCCNsdjg2GsetEwQpDx8+zPe//30+++wzMjIy/D63n6I1bUA+0CpJkg5IwhHUnHJMqUWh1WqVIbL+DBt2xutHO8bHJuyCg02Dys+hdjliIzTcXDqH73yxMOhzyGMIUlJSmDt3rk9BFUmSlPx+YWEhBoOB7u5uRTouLS2Np64tVMRcXjnUxlMfNNCjFoeEBaOjo3z22WekpKQoehgy0R89epQHHniA1157jblz5wZ0Xlm0pqGhgQULFkTiQbQG2APcDnwKXAe8Nx3xCZhiorBYLNTU1FBSUhKwbFjPiH/t4ZNFlE7iK+vzJkUOMkwmE2VlZeTn55OTkxPUOWJiYigoKFDaq2XV5+HhYZKTk7lkbgbXffs8AB7Z8Rlv15mwqmZGSKCR4O5LVrIqN57+/n4l6H7kyBH0ej1vvPEGu3fvZuHChQGfWxatueKKKwCO41m05lngD5Ik1QF9OMhkWiD5IKigbzmr1aoEIWXTW6/Xs3bt2oALU36/95/8vwMmxaKQgNzkaNoHjCGzIGK1cNnCWB66dAGpqamT6jQER7q3srKSxYsXh6W01263MzAwQHd3N319fZjNZlJTU1myZAmvV3Tz+Lt1jJrViGig0OCY3aKV4D82LeaGtXNcfi+EYN++ffzkJz8hMjISSZL405/+xIIFCyZz2Rmf/gobUdhsNqxWK2azmbKyMtLS0hgeHmbBggUBE8XNT73H4e7w3PTrCpJ46NKFrMlLVGr/+/r6iImJISMjg/T09ICFXuWp2StXrgx7o5DZbObIkSNkZWUhhFDK3TMyMvhnh51f/K1J7Y4NANesSmduRiLr5yZ7jEPV1tZy22238ac//YlVq1YxODhIbGxswMptbji7iaK/v5+KigoKCwvJzMykqqqKnJycgGITR1oGuOXZgyEfsLM0K45HNy3xeDMIIRgdHaW7u5uenh4kSSIzM5OMjAyfLlNHRwctLS2sWbPGY3NRKDE2NkZ5eTmLFi0iLS1Ned1sNitCxkajkfv+bsAUeIf/WQetBv5we4nXQHVDQwM333wzL7744jgZhUlixhNF2GIUvb29LvlnCK6DdH9Df0hJYm5KND/btnzCrIUkScTHxxMfH8/8+fMxGo10d3dz/PhxLBYL6enpZGRkuKhDCyFobGxkYGCAkpISvyabTQbDw8NUVlayYsWKcRPWIiMjlaG7NpuNJ+Ja+cZf6sO6ntkOjQT/8aXFXu+L5uZmbrnlFp599tlQk8SsQNju5sTExHGK28EQhfv0r6RoLYUZcfSMmGnqN/p9Hn8Iwhuio6PJz88nPz8fq9VKT0+PUvsvR8P1eockn6f0Z6ghuzZr1qwhNjZ2wmO1Wi0Xr5rLqs+7qWgfnvDYsxnXl+SOi0fIaGtr46abbuLpp5+mtLR0ilc2MxA2opADPS4XC1DlyjH9q9nltUGjjUMtQwGt5e7z8kOSxQDHZ8jOziY7Oxu73U5vby/Hjx9XWpO7u7tJS0sLm0XR1dVFU1MTxcXFAbk2Xy7OUYnCC3Qaia2rPQ+j6uzs5MYbb+TJJ5/kvPPOm+KVzRxMeR1FICpXv/uocdJ9G9uvGh+5DhUsFgsNDQ0UFhaSnZ3N8PAwer2exsZGIiMjlSKqUMUqWlpa0Ov1FBcXBxw8u2HtHP5R18d7Xkrgz1boNBI//NIij5amXq/n+uuv5/HHH2fjxmmZ5DdjMKWVmVqt1m8l7v31PbxXHdxNrZHgosXpAVdQBoLR0VEqKipc0p+JiYkkJiZSWFjI2NgY3d3dVFRUIIRQSCMYcRMhBPX19YyMjFBUVBR06vae8wr4x4mes7aPxPmO1Gnh2qJctq7O9niP9PT0cP311/OTn/yESy+9dOoWOUMRtqyHEAKz2bVISq/XMzQ0RGHhxG7AyMgI97ywnyNBpERzk6L45bUrwkYQAP39/dTU1Pid/jSbzfT09KDX6zEajaSlpZGRkaGoK00EIQTHjx9HkiSWLl066VF5rxxqY/tbtZM6x2yGToJLF8Ryy3kLKZ2f7vGY/v5+rr32Wn74wx+yefPmqVjW2Zv18AR/XI+enh7e+ryKsp7ASUIrEXaS6OzspLm5maKiIr+VnCMjI8nNzSU3NxebzUZvby9tbW0cP36cpKQkMjIyPBZ5yY1z8fHxLFiwICTzNKs6RyZ9jtkG59J+O5CdGIXU28j+7nolgyWL0AwODnL99dfz8MMPTxVJzApMuesxUTCzqamJzs5OhmNyEKLJ72ulx0qsTNVw0dwY0qVhzOaYSU3D9gR5rkhfX9+k0p9arZbMzEwyMzOx2+1KkVddXR2xsbFkZmaSnu540pWXl5OVleV345w/mPGPrjBg0/J03q3uwSogUqvhiqL5FOUnYbFY6OnpobGxkcbGRnbs2EFzczMPPvgg11577XQve0YhbK4HMC4eMTw8rEwPc4Zc4m21Wlm2bBmvHm7nR3tP+HUN5yKZsbEx9Ho93d3dSJJERkYGmZmZAfeVuEPW8bTZbCxbtiws6U+5yEte/9jYGJmZmSxcuDDgGRQT4WjLILe+cPisqtbUaeBbF+Ri1UZ7rbjs7+/njjvuICIigs7OTr7yla/w0EMPTdUSZzx/h9X18CSw625RWCwWysrKSE1NpaCgwOGT+2keX7IolXu+ME/5w8fGxjJv3jzmzZuHyWRSiqSsVivp6elkZmZ6HGo7EWw2GxUVFSQmJjJ//vyQmP+eIBd5SZKEXq9n2bJlWCwWl67RzMxMF53GYFCUn8R/bFrMY2/VhrUtfybBZgerNpqvfcFzh6fBYOD222/n5ptv5u677wbAaPS/RudswLTGKORhwwsXLiQtLQ0hBGVtQ7x2tGPC80jAo5sWceM67yZ5VFSUonchm5gnT57EYDD4HUyU+1TmzJlDbm5uwJ83UAwODlJVVcXKlSuVfpj8/Hxl/Q0NDYyOjpKamkpGRoYi/hooblg7h+OdI4r025mOCK3E+rnJHn9nNBr5l3/5F6677jruuusu5fVgrbi77rqLN998k8zMTCorK8f9XgjBgw8+yN69e4mNjeWFF16gpKQkqGtNJaacKGSLore3l+rqalauXElcnGPOgiRJ7Cnv8pq+k4AN+bHc/8UlFOcn+33diIgIcnJyyMnJGRdMTE5OJjMz00VnAE6nP937KMKF3t5eTpw4QVFR0ThXyXn9drudvr4+urq6qKmpISEhgczMTNLS0gJKm2ZF+m7bn+2YmxzJhoXpXlOgZrOZ22+/nU2bNvH1r389JNbiHXfcwf33389tt93m8fdvv/02J06c4MSJE3z++efcd999fP7555O+brgxpa6HrMTd3NxMR0eHEhSUJdKPtgyy64j3p9zmZSn8/PqiSa3JPZg4MDCAXq+ntrZW2XRarVbp/pyKWQ1yI1lJSYnPIKxGoyE9PZ309HRFDUuv11NfX09UVJTSvObtPLIkn35wLBwfZUbhnIXp/OdVSzz+zmKxcNddd7Fx40YeeOCBkLmUF154IY2NjV5/v3v3bm677TYkSWLDhg0MDAwgSVKOEGJiM3qaMaUWhRACo9FIf38/xcXFSJKk/AfwenmnV2tCp4GbNswP6Xo0Gg2pqamkpqYqm04W+01JSWF4eJjo6OjJthBPiObmZnp6eoLKpDirYS1atEjpeC0rKwNQgrlyP4gQghMnTmC1Wtl8zhJery3DeIZVX8nbPUIrsc1LWbbVauWrX/0qa9eu5Tvf+U7Y4k6e4C6qm5eXR21t7RxAJQo4HbSUJIkVK1YorobzH6m+e9TlPfJvNBOU2YYSAwMD2O12Nm7ciNFoRK/Xc+TIERcrJFTl2EII6urqMBqNFBUVhSSTEhcXR1xcnBLM7enpoaamBpPJRFpaGqOjo0RHR7Ns2TIkSeIHVxTyX3trz6gMyNXLklmYk+o1u2Gz2fjXf/1Xli1bxr/9279NKUnMZoTd9QCHbsLRo0dZsGABJ0+epLOzk4yMjHFPUJPb0y03XsN16wo4Z35qWEnCOf0pb1q5zXzBggWKbqVzObbzkzpQ2O12jh8/jk6nY+XKlWG5WaOiopRWc7PZzNGjR7HZbBgMBqqrq8nIyKB/zBJynY/pRIRG4uYNC7zeKzabjW9+85vk5eWxffv2aSEJd1Hd1tZWGC+qO+MQdouir6+P48ePK0HL6OhoOjs7aWpqIiYmRikwioiIGNfh+NWNhWFr6JLhXP24ZMkSjzePs26lLApTU1OD2WwmLS2NrKwsv9OWNpuN8vJykpOTmTdvXthvVpvNxrFjx8jOzqagoMBFQi92pMdlbutshgRcU+Q5aAkOcv7Od75DSkoKP/3pT6fNktiyZQtPPfUUN910E59//jlJSUnM9PgEhLngqqmpicbGRpdGJtnElguMurq66OnpITIyktTUVF4+0ELlYASbi/LCThJy+jM3N5c5cwK/lqxNodfrlbRlZmYmycnJHm9E2f3KyckJ6nrBrO/o0aNK+bg7hBDc96cj/KN+0MO7Zw8cM0ThOyWRlM53pL6ds1h2u50f/OAHAPz6178Oq17IzTffzAcffEBPTw9ZWVn86Ec/wmJxKKTfe++9CCG4//772bdvH7GxsTz//POsW7duxvs/YSWKgYEBtFrtuKClJ8iS9BEREUr0PpQxAXeEOv0ppy31ej2Dg4MkJSWRmZlJamoqGo0Go9FIWVkZCxYsCGj+Q7CQ3Y25c+e6zHJ1x2yv1NQA1691dIGunpPgIjis1Wo5ePAgTU1NmEwmfvvb34ZdVChInN1E8eKLL7JgwQKfrdHd3d2cPHmSVatWERcXp8QEZNWoUJViyxgYGFDcoXCkP4UQStq1r6+PqKgoRkZGWLFixZTUZMikVFhY6Nf1fvRWzawpvpJwlO3b7Kfk6zwoZYPjb9De3s5DDz3EoUOHWLx4Mffeey833TRtivcTYcYTRVhjFImJifz617+mpqaGSy65hK1bt1JaWurifrS0tNDd3e1SQ+AcEzCZTOj1eqqqqrDZbGRkZJCVlRV0IFGv19PQ0EBxcXFIeyicIUkSKSkppKSk0N/fz7Fjx0hPT6euro7m5maftQ6TgSy4u3TpUpKTk/16z9bV2ewu68RkdUwey4iPoHfEMuNKvLUS/PBLi5AGO2gyRnHZ6rkTBrn/+Mc/kpiYqKSgOzs7p3C1ZxbCalHIMBgM7Nu3j507d1JWVsbGjRu56qqrePPNN7npppsoKSnxyySUA4l6vR6z2Ux6ejpZWVl+9280NzfT3d3N6tWrw1obIaO7u5v6+nrWrFmjkJJ745q/6t7+YGRkhIqKiqAspVcOtbmkSqdr0PNEuGRxGncvsZGenu4+4NcFQgiefPJJjhw5wp///OeQ/K337dvHgw8+iM1m45577lFiHjKam5u5/fbbGRgYwGaz8fOf/5xNmzb5e/oZb1FMCVE4w2Qy8frrr/Pd736XzMxMiouLufbaazn//PMD+oPK/Q96vV7p38jKynJRxlY+hBDU1tZisVhYvnz5lPip7e3ttLW1sWbNGq+Wg6zu3d3d7dK4FswskMHBQY4fP664b4Hi/z5q4sn36mccOThDJ8Hjm+ZwxdrFXo8RQvD000/z0Ucf8corr4TEarPZbCxevJh3332XvLw8SktLeemll1i+fLlyzNe+9jWKi4u57777qKqqYtOmTRNWaLphxhPFlE8zj4qK4tixY/zP//wPmzdv5v3332fXrl1873vfY/369Wzbto2NGzf6/AO79284K2OnpqaSlZVFUlISdrudyspK4uLiWLx48ZSkxRobG+nv76ekpGTC2IyzurfFYlFiNTLxZWZmkpiY6HPNzqrcwVom6+cmo9Uwo2Xy7ECTwXtwWwjBs88+ywcffMCuXbtC5trt37+fwsJCZRrYTTfdxO7du12IQpIkhoYcos+Dg4NT0kQ4lZhyi8IbrFYr//znP3n11Vf58MMPKS4uZtu2bVxyySUBxRJsNptL9sFqtZKVlcWiRYvCbkmEynKRG9f0ej3Dw8OkpKQoaVf3c3Z3d9PQ0BCSgUMP7Kjg7zNUfFcConQanru1yGtc4oUXXmD37t3s3r07pPGnnTt3sm/fPp555hkA/vCHP/D555/z1FNPKcd0dHRw+eWX09/fz+joKH/7299Yu3atv5dQLQp/odPpuPjii7n44oux2Wx88skn7Ny5kx/96EcsX76cbdu2cdlll/k1x0IWsS0rKyMvLw+j0agUtzinLEMJu93OsWPHiI6OZsWKFZOyXNwb1/r7+z12i+r1elpbW4NS5faE9HjXJ3BilJYLF6fxdqV+WtKn8l9Io4EvF3sXwgX405/+xK5du3jjjTfCFqSeCC+99BJ33HEH3/nOd/j000+59dZbqaysnKnp2IAxY4jCGVqtlgsuuIALLrgAu93OgQMHePXVV/n5z39OYWEhW7Zs4corr/QasJN1HZynaMkpy66uLk6cOBF0e7YnWK1WysvLSUtLY+5cz+IowUKj0ZCWlqbodQwODtLd3U11dTVCiKAmaXvD1tXZ7DrSgfVUXfeQycbeSj0l+UlYbTZaekZAqyUqQkditI7artGwZkbswIa8GL552bIJsxuvvvoqf/zjH3nrrbeCzoZNBE9l1+4Fc88++yz79u0D4Nxzz8VoNNLT00NmZmbI1zMdmDGuhz+w2+0cPXqUnTt38vbbb5Ofn8+WLVvYtGmTkgrs6uqisbGR1atXe/XXnduze3t7XbQqA+3glKs78/LyyMnJmexH9AtyDGTBggX09vbS09ODTqdTMiiTcUG2v1XDKzOopuK8BSk885Uir79//fXXefrpp3nzzTdJSgpPP5DVamXx4sX8/e9/Z86cOZSWlvLnP/+ZFStWKMd86Utf4sYbb+SOO+7g+PHjXHrppbS1tflrWc5412NWEYUzhBBUVlayc+dO3nrrLUVNOTIykieeeMJvU1wIwcjIiFJKHh0drWw4X+cwGAxKYZMsiBtOyB2nJpNpXAzEYDAoadfJNK7NNDn/71+Sz+3nL/S44d566y3++7//m7feeiugwdfBYO/evXzrW9/CZrNx11138e///u88+uijrFu3ji1btlBVVcVXv/pVRkZGkCSJX/ziF1x++eX+nl4liqmAzWbja1/7GocPHyYqKoqEhAS2bNnC5s2bycjICChe4Nx/Ij+lMzMzx0XQ5SHBy5cvD9uTzBlCCKqrq5EkyWvzmgyz2ayQhlxv4q/e5v991MSv36/3q6tUK8G8tFh0WonartFxN4tWgkWZcVR3jXp8/0SQgJuK0rl2oYahoaFxYw3++te/8rOf/Yy9e/dOSbVrmKESxVRgeHiYZ599lm9+85tIksTJkyfZtWsXu3fvJjIyki1btrB161ays7MDIg3n4iiNRqM8pQ0GAzU1NUHXLAQKu91OVVUV0dHRLFzo+enqDc6Na2NjY0rjmje90KMtg9zx+yOYT0UvJQkuWZzOBYWpHO8c4WT3KB2DRkxWO32jFgSebxIJWJIVxwn9aMCBUHnMn1yaLceXuru72bFjBx9++CGdnZ28/fbbLFniWcFqlkEliumEEILm5mZ27drF66+/jt1uZ/PmzWzbto28vLyANpwsZNPW1obBYKCgoIA5c+aErP/EG2QVcLktfbLnklPH8lPaUxbo314r5/XKXsCReViRm8CynAQSorS88GlLyDIgGgmK85M41Hy6e3VVbgKPXOFdpOjDDz9k+/btXHLJJfzjH//gkUce4eqrrw7NgqYPKlHMFAgh6OjoYNeuXfzlL3/BYDBw1VVXsXXrVr+ncLW2ttLZ2cmyZcvo7+9Hr9djtVoVSyPU1oXVaqWsrCzkQ4AAF12Kvr4+4uPjFb3QfQdr+NVhK+ZTvR/+QOJ02bc/75GAu87L55Il6dz+4hFsdod83Qu3FXsliU8//ZTvfve7vPnmm0rWQVZKCwa+yrIBXnnlFUXkZs2aNfz5z38O6lo+oBLFTIVer+cvf/kLr732Gn19fWzatIlt27Z5rN6UBWmHhoZYtWqVSzpVrqjs6urCZDIppDHZ+RsWi4WjR49OSTZFCMHw8DBNTU3o9XpSUlI41B/Nk590+YxVaCW449x8GnoNvF/TE9ANE63T8OiFjiKyTlsC6+eleCWJgwcP8s1vfpM9e/ZQUFAQwFU8w5+y7BMnTnDDDTfw3nvvkZKSgl6vD1e6UyWK2YDe3l52797Nrl276Ozs5IorruCaa65h2bJl2O12KioqiIqKYunSpRMW0LjHAwIpw3aGyWRSpAOnQrsCUOaeFBcXY7FYeOrvtbx4dGDccRKnXYbEmAjlDv+gtidgl0QC/mVVAo9sWzvh93P06FHuu+8+/vKXvyhl1JPFp59+yvbt23nnnXcA+NnPfgbAI488ohzz/e9/n8WLF3PPPfeE5JoTYMYTxYwsuJpqpKWlcdddd3HXXXcxMDDAG2+8wU9/+lNOnjyJTqfj0ksv5dFHH/VZZafT6cjOziY7O1spw25paWF4eNin+pUMOeW6ePFiUlNTQ/1RPUIuAy8uLiYyMpLIyEiuKF7AH8tcBW0WJ2s4d14iuRlJ7G8e5f2aHo8FV/Knc+YNzSnJPffXvrSucMLvo7KyknvvvZedO3eGjCTAsxq2+3yN2lpHmvj888/HZrOxfft2rrzyypCtYTZBJQo3JCcnc+utt7Jt2za2bt3KvHnzaGxs5Pzzz+fSSy9l69atrFu3zidpuJdh9/X10d7eTnV1tc+hQ8uWLZuSlCs4CtSam5vHlYEX5SexcXE67zn1fli1UdT1W/hjWZNH68FbnEIjwUWL0/mgtkfR55RwiM5MNMjp+PHj3HPPPbz88sssXuy9YzRcsFqtnDhxgg8++IDW1lYuvPBCJbB8tkElCi/QaDT84Ac/UIpmZE2N3/3udzzwwANs3LiRrVu3smHDBp8l4M5De+QgYldXF7W1tSQmJpKZmUlERITSJh5Mm3kw6OzspKWlhaKiIo/FZfecV8BHdb1KqrS+10B9r+dzaYCClEia+83jSOLOc/MZMdnQaSQsNjGhMpWM2tpa7rzzTv70pz+5xA1CBX/KsvPy8jjnnHOIiIhg/vz5LF68mBMnTlBaWhry9cx0hCRG4St6bDKZuO222zh06BBpaWns2LFj0qm+6YTRaOTdd99l586dHDp0iPPOO49rrrmG888/P6AScLl3QxbUSU1NJTc3l/T09En3n/hCe3s77e3tFBUVTbjmVw618djeWq9BTQ2wKDOWE91j446RSeJP+9sw2+xoJbhoXgx3bFw6oSXR0NDALbfcwgsvvEBxcXHgH84P+FOWvW/fPl566SVefPFFenp6KC4u5ujRo+Eo8DrzYxQ2m41vfOMbLtHjLVu2uDwFnn32WVJSUqirq+Pll1/m4YcfZseOHZO99LQhOjqazZs3s3nzZsxmM++//z47d+500dS48MILfeohSJKE1WrFYDBw3nnnKRWVDQ0NyigDT/NPJou2tjY6OzspLi72SUgDBus4OX+dBjYuSictPpKEKC3Pf9riQhJywPPyAi0H6k5L7CFgxdzsCUmiubmZW265hWeeeSZsJAGOeNJTTz3FFVdcoZRlr1ixwqUs+4orruCvf/0ry5cvR6vV8vjjj58JVaBBYdIWhT/R4yuuuILt27dz7rnnYrVayc7OVqTgziRYrVb+8Y9/8Oqrr/LPf/5T0dS4+OKLPbY+d3V10dTURFFRkQupeBplkJWV5Vf/iS/IGqVr1qzxy2pxr9TUAI9e5XAbPFkbWgm+XJJLQpSWFz9rVTpRJSBCA7/YlMfGFQUeG9fa2tq44YYb+M1vfsN55503qc85yzDjN8KkH1X+RI+dj9HpdCQlJdHb2zsljVRTCZ1OxyWXXMIll1yCzWbj448/ZteuXWzfvp0VK1awbds2vvjFLxIbG8uxY8cwGo0eZ45KkqRMKlu4cCGjo6PKeMPJdIk2NzfT19cX0AjDovwkthXl8OqhdiVIWdU54pEk5NjD4sx4bnvxiEISAOcuSOGeDTnkRBgpLy8HUIK9MTExdHZ2cuONN/Lkk0+ebSQxK6AGM8MErVbLhRdeyIUXXojdbmf//v3s3LmTn/3sZ8THx6PVatmxY4dfbkVcXBzz589n/vz5SpdoeXm5Is6bmZnpU6ylsbGRwcFBVq9eHbCYyrbV2bx+tAOzTSCAnYfbQeCSGtVI8OgmR2biB69XuZCETiNx/8b5SjHV3LlzMZlMdHd38/HHH/P9738fu93O9773PS644IKA1qZiajBpovAneiwfk5eXh9VqZXBw8Kzy9TQaDRs2bGDDhg089thjfPTRRxQXF7Np0yYKCgoUTQ1/UqIxMTHMnTtX2Wx6vZ5jx45hs9kU0nBvLa+vr2dkZIRVq1YFpbhUlJ/ENUU5yuwPb0HLNyu6OOjUtwGnG7zcKy6joqLIy8sjOjqa5ORkNm7cyF//+lcOHTrEb3/724DXqCK8mHSMwp/o8W9+8xsqKir47W9/y8svv8xrr73GK6+8EoLlzz7s27ePyy67DK1Wq2hqvPrqq+zdu5eMjAy2bt3K1VdfHXCxlTzKoKurC4vFQkZGBhkZGXR1dWE0Gictz+dJp8IxjEfi9g15/OHzViWOIaMgJZqfb1vutSy7v7+fa6+9lh/+8Ids3rwZmFzvBvjXvwGwa9currvuOg4cOMC6deuCvl6IMONjFJMW9HOOHi9btowbbrhBiR7v2bMHgLvvvpve3l4KCwt54okn+PnPf+7zvPv27WPJkiUUFhZ6PP6JJ55g+fLlrF69mksvvZSmpqbJfpQpwZVXXqkEESVJYtWqVTz22GMcOHCAJ554gu7ubr785S+zbds2nnvuOfR6PT7IHIDIyEjmzJlDSUkJJSUlREdHU15eTmtrK1FRUQwPD/t1Hm843jni8rNGcozy+/3txSRER2DxUIF113kFXklicHCQ66+/nocfflghCWBSJCFn4N5++22qqqp46aWXqKqqGnfc8PAwTz75JOecc07Q1zrbMCN7Pfxp2Hn//fc555xziI2N5emnn+aDDz6Y1SlXZwghXDQ1oqKi2Lx5s9+aGrIauN1up7CwkL6+Prq6uhgdHVX6T7zpUXiDu0Te2oIk/nBHCUdbBnm9vJPXnLQ25c7Q73yx0OO5hoeHue6667j//vu58cYb/V6DL/iTgQP41re+xWWXXcbjjz/OL3/5S9Wi8AMzUiLYeY5CZGSkMkfBGRdffLHii2/YsIHW1tbpWGpYIEkShYWFPPzww3z88ce88MILANxxxx1ceeWV/O///i8tLS0eLQRZCQtg6dKlREREkJWVxerVq1m/fj0pKSm0trby2WefUV1dTV9fn1+WxrbV2eic7paKtiF+9bc6bnvxCDsPtyMhWJcdwQ0lOfzpzhKvJDE6OspNN93E1772tZCSBHjOwLW1tbkcc/jwYVpaWrjqqqtCeu0zHTMy6+FPytUZzz77LF/60pemYmlTDkmSmDt3Lg899BDf/va3FU2Ne++9F6PRyNVXX83WrVuZP38+drud8vJy4uPjKSwc32wljzLIyMhQxgB0dnZSU1NDUlISWVlZ4/pPZBTlJ/Hl4lxeOZUmtdgEz33SopicQsD5y/L4+gXzvH4Wg8HATTfdxK233sqtt94aui/JT9jtdh566CGFeIPFZOMosxEzkigCwR//+EcOHjzIhx9+ON1LCTskSSI3N5cHHniA+++/X9HUeOihh+jv70en03HRRRfx7//+7z5vZPcxALIQT21tLQkJCWRlZSn6lDLkYcZmm31c5kOrkThnnneBW6PRyL/8y79w/fXXc+edd07qe/AGXxk4Wef0oosuAhy9Llu2bGHPnj0BuR/yd3vgwAFGR0dZv359WMYEzCTMyBiFv77m3/72Nx544AE+/PDDM2Z+QjCwWq3ceOON2O12zGYzXV1dLpoagTz95P4TeZRBXFwcWVlZSv/J0ZZBnvqwgU/r+5WbQ66h8NbkZTab+cpXvsLll1/OAw88ELansT8ZOGdcdNFFAcUoqqqqiIqKYuHChezevZtXX32VDRs2kJqayi233DKZpc9482RGWhSlpaWcOHGChoYG5syZw8svvzxOguzIkSN8/etfZ9++fWc1SYDDpN+6dSu33XYbAAMDA+zZs4cf//jHNDU1cdlll7Ft2za/iq0kSSI5OZnk5GRF+UruP4mOjiYrK4t7z8/nUNMAZptA6yaE6w6LxcKdd97JRRddFFaSAP/6N4KBEAKDwcB///d/89WvfpXc3Fyam5t5/PHHaWtro66uDnB81lBMbJuJmJEWBfieo/DFL36RiooKRSauoKBAScd6wyzNsU8Kw8PDvPXWW+zatYva2lpFU2Pt2rUBF1+NjIyg1+tpb2+nts9CrzaVi5bPoXS+51J8q9XK3XffTXFxMY888sis9+sffvhhSktLue666xgdHSUuLo76+no++eQTBgcHWbFiheLWBIgZ/8XMWKIINfxJuYJjY1111VWYzWaeeuqpWU8UzhgbG+Ptt99m165dVFZWKpoa55xzjt9t7R0dHbS3t7N48WJlkLKzSI/cf2Kz2bj33nspLCxUxGlnI+rr6/n5z39OSkoKo6OjfOELX+Cmm25Sfl9WVsbNN9/M3XffzXe+851gLzPjv5yzhihmcY49LHDW1Dh8+LCiqXHeeed57T+RSaKoqMiFWORRBnq9nv7+fj755BNaWlqYO3cuP/3pT2ctSYBDS+Xll19mZGSEp556itHRUb73ve8xd+5cLrvsMjo7O3njjTf45je/CQSdEZnxX9CMjFGEA/6kXJ1z7I8//vhUL3FK4a6p8d5777Fr1y6++93vcs4557Bt2zYuuOACpf29s7OTtra2cSQhn6ugoICCggL0ej3PPfccFRUVNDY28vzzz3PXXXdNx0ecNIQQREVFcfvttwOO9HJlZSUNDQ189NFHJCQkcPHFFyskYbPZwi44NF04a4jCF0KVY5+NiIyM5Morr+TKK6900dR45JFHKCkpISsri+HhYX7xi19M2O1qt9v55S9/SX5+Pn/5y18YGBigsbFx0uvzFVt64okneOaZZ9DpdGRkZPDcc8+FZKq8bBnIVkJERASpqak89thj2O32cTGeM5UkYIZWZoYDgeTY582bx2effcaWLVs4ePDgdCx32iBrajz99NOKGvjLL7/M559/zte//nX27NnD2NjYuPfZ7XYeffRRzGYzv/71r9FoNKSmplJSUjKp9fjTv1FcXMzBgwcpLy/nuuuu4/vf//6krukOmTBWr17N6KhjjmowXbizGWfNp3VOuZrNZl5++WWXdFlSUhI9PT00NjbS2NjIhg0bAi7EOdMghKC+vp7KykoOHTrEgw8+yIEDB7j00ku57bbbeO211xgZGUEIwY9//GP6+vp4+umnQ7qJZlI5v9FoZNWqVWE590zHWUMU/nS5BgNfXa7gGEu3fPlyVqxYMdnCnCmFTqfjueeeIykpSdHU+NWvfsWRI0d45JFHqKys5IorrmD9+vXU1NTw7LPPhtz89qd/wxnhLOe/4IILuOOOO8Jy7hkPIcRE/6mYAFarVSxYsECcPHlSmEwmsXr1anHs2DGXY2pra0VRUZHo6+sTQgjR1dU1HUsNG2w2m9i9e7cYHh4Oy/lfffVVcffddys///73vxff+MY3PB77hz/8QZxzzjnCaDSGZS1hhK99OO3/nTUWRTjgj1n8u9/9jm984xukpDj6IM60KlKNRsOWLVvCNovEHwU1cJTz/+QnP2HPnj0Ba4mq8A2VKCYBf8zi2tpaamtrOf/889mwYQP79u2b6mXOaviKLcHpcv49e/accUQ8U6CmR8MMdSzd5OBP/8b3vvc9RkZGuP766wH/yvlVBAaVKCYBdSzd1GDTpk1s2rTJ5bXHHntM+fff/va3qV7SWQfV9ZgE/DGLt23bxgcffABAT08PtbW1IZ3KrULFVEAliknAn5TrFVdcQVpaGsuXL+fiiy/2eyydr7Rrc3MzF198McXFxaxevZq9e/eG/POpUCHjrGkKm03wp9P1a1/7GsXFxdx3331UVVWxadOmkJRLq5gWzPimMNWimIHwJ+0qSRJDQ0OAQ/o+Nzd3Opaq4iyBShQzEP6kXbdv384f//hH8vLy2LRpE//7v/871csMGr7cKpPJxI033khhYSHnnHOOainNAJy1RCG7XA899JAyNHc24aWXXuKOO+6gtbWVvXv3cuutt2K3232/cZrhT5PXs88+S0pKCnV1dXz729/m4YcfnqbVqpBx1hKF3BGYlpbGO++8g91u5w9/+AOvvvrqtG84f9Kuzz77LDfccAMA5557LkajkZ6enildZzDwx63avXu3ogFx3XXX8fe//31SU85UTB5nLVFYrVbA0Tr8xhtv8MQTT7Bz505iY2PRaDTYbLZpIwx/0q4FBQX8/e9/B+D48eMYjUYyMjKmY7kBwR+3yvkYnU5HUlISvb29U7pOFa44a4lCp9NhNBoVrYUFCxawY8cOrrrqKoxGI1qtdto0B/xJu/7qV7/id7/7HWvWrOHmm2/mhRdemNWScypmNnylR89ISJKUAXwB+AGwG9gKXCCEMEuSlAo8CCQB24UQA9O20BBDkqTngKsBvRBipYffS8CTwCZgDLhDCHE4xGs4F8f3esWpnx8BEEL8zOmYd04d86kkSTqgE8gQZ+PNOkNwtloU64ErgDuBp4EOIBtACNEH/A+w6EwiiVN4Abhygt9/CVh06r+v4fhuQo0DwCJJkuZLkhQJ3AS4N2bsAW4/9e/rgPdUkphenJVEIYR4SwhxrxCiSgjRj4MoLnU65AtAGYAkSWfMdySE+AfQN8EhW4Hfn9JI+AxIliQpJ8RrsAL3A+8Ax4FXhBDHJEl6TJIkORDzLJAmSVId8BAOy0/FNOKsbAqTJEkrhLDJPwsh7jtl4iJJUgRwPvCq/OtpWOJ0YQ7Q4vRz66nXOkJ5ESHEXmCv22uPOv3bCFwfymuqmBzOmKdlIHAmiVN+OUIIqyRJacBvgNuA3FOvn01EoUKFR5yVFoUznIlACNELfE2SpOVA3PStatrQBuQ7/Zx36jUVZznOSovCG5ysiyohxIHpXs80YA9wm+TABmBQCBFSt0PF7MRZb1E440x3MyRJegm4CEiXJKkV+E8gAkAI8VsccYNNQB2O9Oid07NSFTMNZ2UdhQoVKgKD6nqoUKHCJ1SiUKFChU+oRKFChQqfUIlChQoVPqEShQoVKnxCJQoVKlT4hEoUKlSo8AmVKFSoUOET/z8cGasalb9EZAAAAABJRU5ErkJggg==",
-      "text/plain": [
-       "<Figure size 432x288 with 1 Axes>"
-      ]
-     },
-     "metadata": {
-      "needs_background": "light"
-     },
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Mean residual: 0.0015693869\n",
-      "L2 relative error: 0.000965834558042597\n"
-     ]
-    }
-   ],
-   "source": [
-    "if __name__ == \"__main__\":\n",
-    "\n",
-    "    # Problem parameters:\n",
-    "    a = 0.4 # Thermal diffusivity\n",
-    "    L = 1 # Length of the bar\n",
-    "    n = 1 # Frequency of the sinusoidal initial conditions\n",
-    "\n",
-    "    # Generate a dataset with the exact solution (if you dont have one):\n",
-    "    gen_exact_solution()\n",
-    "\n",
-    "    # Solve the equation:\n",
-    "    main()\n",
-    "    "
-   ]
-  }
- ],
- "metadata": {
-  "accelerator": "GPU",
-  "colab": {
-   "collapsed_sections": [],
-   "name": "heat_conduction_1d_uniform_bar",
-   "provenance": []
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.0"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

From f0856169ce6698934ece1b01586d21b3ec2d8ee5 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 16:43:52 -0400
Subject: [PATCH 002/170] tiny change

---
 docs/demos/pinn_forward/heat.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/demos/pinn_forward/heat.rst b/docs/demos/pinn_forward/heat.rst
index f1b248e91..95afea6e9 100644
--- a/docs/demos/pinn_forward/heat.rst
+++ b/docs/demos/pinn_forward/heat.rst
@@ -47,6 +47,7 @@ Next, we define the parameters of the equation:
 
     a = 0.4
     L = 1
+    n = 1
 
 Next, we express the PDE residual of the Heat equation:
 

From 21a19e002d7ac414e9183780722ebbf8cd937be4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 16:47:50 -0400
Subject: [PATCH 003/170] tiny change in Heat.py

---
 examples/pinn_forward/Heat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/pinn_forward/Heat.py b/examples/pinn_forward/Heat.py
index a146693ea..213cca441 100644
--- a/examples/pinn_forward/Heat.py
+++ b/examples/pinn_forward/Heat.py
@@ -3,7 +3,7 @@
 import numpy as np
 
 import deepxde as dde
-##from deepxde.backend import tf
+from deepxde.backend import tf
 
 def heat_eq_exact_solution(x, t):
     """

From df57b3928b4f225b5a2f9c20c99c7997b6e82659 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 17:43:34 -0400
Subject: [PATCH 004/170] tiny change in pinn_forwad.rst

---
 docs/demos/pinn_forward.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demos/pinn_forward.rst b/docs/demos/pinn_forward.rst
index cbc792dcd..83b2d1e2b 100644
--- a/docs/demos/pinn_forward.rst
+++ b/docs/demos/pinn_forward.rst
@@ -35,12 +35,12 @@ Time-dependent PDEs
    :maxdepth: 1
 
    pinn_forward/burgers
+   pinn_forward/heat
    pinn_forward/diffusion.1d
    pinn_forward/diffusion.1d.exactBC
    pinn_forward/diffusion.1d.resample
    pinn_forward/burgers.rar
 
-- `Heat equation <https://github.com/lululxvi/deepxde/blob/master/examples/pinn_forward/heat_conduction_1d_uniform_bar.ipynb>`_
 - `Diffusion-reaction equation <https://github.com/lululxvi/deepxde/blob/master/examples/pinn_forward/diffusion_reaction.py>`_
 - `Allen-Cahn equation <https://github.com/lululxvi/deepxde/blob/master/examples/pinn_forward/Allen_Cahn.py>`_
 - `Beltrami flow <https://github.com/lululxvi/deepxde/blob/master/examples/pinn_forward/Beltrami_flow.py>`_

From a86a54c9c4908f4e35660c95ba6d8cc564046553 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 18:10:01 -0400
Subject: [PATCH 005/170] add heat.py and heat.rst for Deepxde Docs plan

---
 docs/demos/pinn_forward/heat.rst |  40 +++++---
 examples/pinn_forward/Heat.py    | 151 ++++++++++++++-----------------
 2 files changed, 95 insertions(+), 96 deletions(-)

diff --git a/docs/demos/pinn_forward/heat.rst b/docs/demos/pinn_forward/heat.rst
index 95afea6e9..0526f9639 100644
--- a/docs/demos/pinn_forward/heat.rst
+++ b/docs/demos/pinn_forward/heat.rst
@@ -1,14 +1,14 @@
-Heat equation in 1D with Dirichilet boundary conditions and periodic(sinusoidal) initial conditions
-======================================================================================================
+Heat equation
+=============
 
 Problem setup
---------------
+-------------
 
 We will solve a Heat equation:
 
 .. math:: \alpha \frac{\partial^2u}{\partial x^2}-\frac{\partial u}{\partial t} = 0, \qquad x \in [-1, 1], \quad t \in [0, 1]
 
-where :math:`\alpha` is the thermal diffusivity constant.
+where :math:`\alpha=0.4` is the thermal diffusivity constant.
 
 With Dirichlet boundary conditions:
 
@@ -18,20 +18,20 @@ and periodic(sinusoidal) inital condition:
 
 .. math:: u(x,0) = \sin (\frac{n\pi x}{L}),\qquad 0<x<L, \quad n = 1,2,.....
 
-where :math:`L` is the length of the bar, :math:`n` is the frequency of the sinusoidal initial conditions.
+where :math:`L=1` is the length of the bar, :math:`n=1` is the frequency of the sinusoidal initial conditions.
 
-The exact solution is :math:`u(x,t) = \exp{\frac{-n ^2\pi ^2 \alpha t}{L^2}}\sin (\frac{n\pi x}{L}))`.
+The exact solution is :math:`u(x,t) = e^{\frac{-n ^2\pi ^2 \alpha t}{L^2}}\sin (\frac{n\pi x}{L})`.
 
 Implementation
------------------
+--------------
+
 This description goes through the implementation of a solver for the above described Heat equation step-by-step.
 
-First, the DeepXDE and TensorFlow (``tf``) modules are imported:
+First, the DeepXDE are imported:
 
 .. code-block:: python
 
     import deepxde as dde
-    from deepxde.backend import tf
 
 We begin by defining a computational geometry and time domain. We can use a built-in class ``Interval`` and ``TimeDomain`` and we combine both the domains using ``GeometryXTime`` as follows
 
@@ -56,7 +56,7 @@ Next, we express the PDE residual of the Heat equation:
     def pde(x, y):
         dy_t = dde.grad.jacobian(y, x, i=0, j=1)
         dy_xx = dde.grad.hessian(y, x, i=0, j=0)
-        return dy_t - a*dy_xx
+        return dy_t - a * dy_xx
 
 The first argument to ``pde`` is 2-dimensional vector where the first component(``x[:,0]``) is :math:`x`-coordinate and the second componenet (``x[:,1]``) is the :math:`t`-coordinate. The second argument is the network output, i.e., the solution :math:`u(x,t)`, but here we use ``y`` as the name of the variable.
 
@@ -65,14 +65,24 @@ Next, we consider the boundary/initial condition. ``on_boundary`` is chosen here
 .. code-block:: python
 
     bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)
-    ic = dde.icbc.IC(geomtime, lambda x: np.sin(n*np.pi*x[:, 0:1]/L), lambda _, on_initial: on_initial)
+    ic = dde.icbc.IC(
+        geomtime,
+        lambda x: np.sin(n * np.pi * x[:, 0:1] / L),
+        lambda _, on_initial: on_initial,
+    )
 
 Now, we have specified the geometry, PDE residual, and boundary/initial condition. We then define the ``TimePDE`` problem as
 
 .. code-block:: python
 
-     data = dde.data.TimePDE(
-        geomtime, pde, [bc, ic], num_domain=2540, num_boundary=80, num_initial=160, num_test=2540
+    data = dde.data.TimePDE(
+        geomtime,
+        pde,
+        [bc, ic],
+        num_domain=2540,
+        num_boundary=80,
+        num_initial=160,
+        num_test=2540,
     )
 
 The number 2540 is the number of training residual points sampled inside the domain, and the number 80 is the number of training points sampled on the boundary. We also include 160 initial residual points for the initial conditions.
@@ -105,7 +115,7 @@ After we train the network using Adam, we continue to train the network using L-
     losshistory, train_state = model.train() 
 
 Complete code
---------------
+-------------
 
-.. literalinclude:: ../../../examples/pinn_forward/Heat.py
+.. literalinclude:: ../../../examples/pinn_forward/heat.py
   :language: python
diff --git a/examples/pinn_forward/Heat.py b/examples/pinn_forward/Heat.py
index 213cca441..461aab1f8 100644
--- a/examples/pinn_forward/Heat.py
+++ b/examples/pinn_forward/Heat.py
@@ -1,33 +1,27 @@
-# Import libraries:
+import deepxde as dde
 import matplotlib.pyplot as plt
 import numpy as np
 
-import deepxde as dde
-from deepxde.backend import tf
 
 def heat_eq_exact_solution(x, t):
-    """
-    Returns the exact solution for a given x and t (for sinusoidal initial conditions).
-    
+    """Returns the exact solution for a given x and t (for sinusoidal initial conditions).
+
     Parameters
     ----------
     x : np.ndarray
     t : np.ndarray
     """
-    
-    return np.exp(-(n**2*np.pi**2*a*t)/(L**2))*np.sin(n*np.pi*x/L)
+    return np.exp(-(n**2 * np.pi**2 * a * t) / (L**2)) * np.sin(n * np.pi * x / L)
+
 
 def gen_exact_solution():
-    """
-    Generates exact solution for the heat equation for the given values of x and t.
-    """
-    
+    """Generates exact solution for the heat equation for the given values of x and t."""
     # Number of points in each dimension:
     x_dim, t_dim = (256, 201)
 
     # Bounds of 'x' and 't':
-    x_min, t_min = (0, 0.)
-    x_max, t_max = (L, 1.)
+    x_min, t_min = (0, 0.0)
+    x_max, t_max = (L, 1.0)
 
     # Create tensors:
     t = np.linspace(t_min, t_max, num=t_dim).reshape(t_dim, 1)
@@ -37,85 +31,80 @@ def gen_exact_solution():
     # Obtain the value of the exact solution for each generated point:
     for i in range(x_dim):
         for j in range(t_dim):
-            usol[i][j] = heat_eq_exact_solution(x[i],t[j])
+            usol[i][j] = heat_eq_exact_solution(x[i], t[j])
 
     # Save solution:
-    np.savez('heat_eq_data', x=x, t=t, usol=usol)
+    np.savez("heat_eq_data", x=x, t=t, usol=usol)
     # Load solution:
-    data = np.load('heat_eq_data.npz')
+    data = np.load("heat_eq_data.npz")
 
 
 def gen_testdata():
-    """
-    Import and preprocess the dataset with the exact solution.
-    """
-    
+    """Import and preprocess the dataset with the exact solution."""
     # Load the data:
-    data = np.load('heat_eq_data.npz')
-    
+    data = np.load("heat_eq_data.npz")
     # Obtain the values for t, x, and the excat solution:
     t, x, exact = data["t"], data["x"], data["usol"].T
-    
     # Process the data and flatten it out (like labels and features):
     xx, tt = np.meshgrid(x, t)
     X = np.vstack((np.ravel(xx), np.ravel(tt))).T
     y = exact.flatten()[:, None]
-    
     return X, y
 
-def main():
-
-    def pde(x, y):
-        """
-        Expresses the PDE residual of the heat equation.        
-        """
-        dy_t = dde.grad.jacobian(y, x, i=0, j=1)
-        dy_xx = dde.grad.hessian(y, x, i=0, j=0)
-        return dy_t - a*dy_xx
-
-    # Computational geometry:
-    geom = dde.geometry.Interval(0, L)
-    timedomain = dde.geometry.TimeDomain(0, 1)
-    geomtime = dde.geometry.GeometryXTime(geom, timedomain)
-
-    # Initial and boundary conditions:
-    bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)
-    ic = dde.icbc.IC(
-        geomtime, lambda x: np.sin(n*np.pi*x[:, 0:1]/L), lambda _, on_initial: on_initial
-        )
-
-    # Define the PDE problem and configurations of the network:
-    data = dde.data.TimePDE(
-        geomtime, pde, [bc, ic], num_domain=2540, num_boundary=80, num_initial=160, num_test=2540
-    )
-    net = dde.nn.FNN([2] + [20] * 3 + [1], "tanh", "Glorot normal")
-    model = dde.Model(data, net)
-
-    # Build and train the model:
-    model.compile("adam", lr=1e-3)
-    model.train(epochs=20000)
-    model.compile("L-BFGS")
-    losshistory, train_state = model.train()
-
-    # Plot/print the results
-    dde.saveplot(losshistory, train_state, issave=True, isplot=True)
-    X, y_true = gen_testdata()
-    y_pred = model.predict(X)
-    f = model.predict(X, operator=pde)
-    print("Mean residual:", np.mean(np.absolute(f)))
-    print("L2 relative error:", dde.metrics.l2_relative_error(y_true, y_pred))
-    np.savetxt("test.dat", np.hstack((X, y_true, y_pred)))
-
-if __name__ == "__main__":
-
-    # Problem parameters:
-    a = 0.4 # Thermal diffusivity
-    L = 1 # Length of the bar
-    n = 1 # Frequency of the sinusoidal initial conditions
-
-    # Generate a dataset with the exact solution (if you dont have one):
-    gen_exact_solution()
-
-    # Solve the equation:
-    main()
-    
\ No newline at end of file
+
+# Problem parameters:
+a = 0.4  # Thermal diffusivity
+L = 1  # Length of the bar
+n = 1  # Frequency of the sinusoidal initial conditions
+
+# Generate a dataset with the exact solution (if you dont have one):
+gen_exact_solution()
+
+
+def pde(x, y):
+    """Expresses the PDE residual of the heat equation."""
+    dy_t = dde.grad.jacobian(y, x, i=0, j=1)
+    dy_xx = dde.grad.hessian(y, x, i=0, j=0)
+    return dy_t - a * dy_xx
+
+
+# Computational geometry:
+geom = dde.geometry.Interval(0, L)
+timedomain = dde.geometry.TimeDomain(0, 1)
+geomtime = dde.geometry.GeometryXTime(geom, timedomain)
+
+# Initial and boundary conditions:
+bc = dde.icbc.DirichletBC(geomtime, lambda x: 0, lambda _, on_boundary: on_boundary)
+ic = dde.icbc.IC(
+    geomtime,
+    lambda x: np.sin(n * np.pi * x[:, 0:1] / L),
+    lambda _, on_initial: on_initial,
+)
+
+# Define the PDE problem and configurations of the network:
+data = dde.data.TimePDE(
+    geomtime,
+    pde,
+    [bc, ic],
+    num_domain=2540,
+    num_boundary=80,
+    num_initial=160,
+    num_test=2540,
+)
+net = dde.nn.FNN([2] + [20] * 3 + [1], "tanh", "Glorot normal")
+model = dde.Model(data, net)
+
+# Build and train the model:
+model.compile("adam", lr=1e-3)
+model.train(epochs=20000)
+model.compile("L-BFGS")
+losshistory, train_state = model.train()
+
+# Plot/print the results
+dde.saveplot(losshistory, train_state, issave=True, isplot=True)
+X, y_true = gen_testdata()
+y_pred = model.predict(X)
+f = model.predict(X, operator=pde)
+print("Mean residual:", np.mean(np.absolute(f)))
+print("L2 relative error:", dde.metrics.l2_relative_error(y_true, y_pred))
+np.savetxt("test.dat", np.hstack((X, y_true, y_pred)))

From f21d706e17d26d845dc32d8ce6ffbbf6beb9547d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 18:34:48 -0400
Subject: [PATCH 006/170] add heat.py and heat.rst for Deepxde Docs plan

---
 examples/pinn_forward/{Heat.py => heat.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename examples/pinn_forward/{Heat.py => heat.py} (100%)

diff --git a/examples/pinn_forward/Heat.py b/examples/pinn_forward/heat.py
similarity index 100%
rename from examples/pinn_forward/Heat.py
rename to examples/pinn_forward/heat.py

From 663fcda61b6041cde4bf45c120dee45efd3460a3 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 27 May 2022 18:42:52 -0400
Subject: [PATCH 007/170] tiny change in heat.rst

---
 docs/demos/pinn_forward/heat.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demos/pinn_forward/heat.rst b/docs/demos/pinn_forward/heat.rst
index 0526f9639..e4891037f 100644
--- a/docs/demos/pinn_forward/heat.rst
+++ b/docs/demos/pinn_forward/heat.rst
@@ -6,7 +6,7 @@ Problem setup
 
 We will solve a Heat equation:
 
-.. math:: \alpha \frac{\partial^2u}{\partial x^2}-\frac{\partial u}{\partial t} = 0, \qquad x \in [-1, 1], \quad t \in [0, 1]
+.. math:: \frac{\partial u}{\partial t}=\alpha \frac{\partial^2u}{\partial x^2}, \qquad x \in [-1, 1], \quad t \in [0, 1]
 
 where :math:`\alpha=0.4` is the thermal diffusivity constant.
 

From 8b5b9803bc334a0c8df11c527302b78833952060 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 14:57:00 -0400
Subject: [PATCH 008/170] Update demo document for lorenz with exogenous input.
 inverse

---
 docs/demos/pinn_inverse.rst                   |   2 +-
 .../lorenz with exogenous input.inverse.rst   | 150 ++++++++++++++++++
 2 files changed, 151 insertions(+), 1 deletion(-)
 create mode 100644 docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst

diff --git a/docs/demos/pinn_inverse.rst b/docs/demos/pinn_inverse.rst
index 299c602bf..0f69dea30 100644
--- a/docs/demos/pinn_inverse.rst
+++ b/docs/demos/pinn_inverse.rst
@@ -10,8 +10,8 @@ ODEs
    :maxdepth: 1
 
    pinn_inverse/lorenz.inverse
+   pinn_inverse/lorenz with exogenous input.inverse
 
-- `Inverse problem for the Lorenz system with exogenous input <https://github.com/lululxvi/deepxde/blob/master/examples/pinn_inverse/Lorenz_inverse_forced.py>`_
 
 Time-independent PDEs
 ---------------------
diff --git a/docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst b/docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst
new file mode 100644
index 000000000..5eb66faec
--- /dev/null
+++ b/docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst	
@@ -0,0 +1,150 @@
+Inverse problem for the Lorenz system with exogenous input
+==========================================================
+
+Problem setup
+-------------
+
+We will solve the Lorenz system:
+
+.. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - 10\sin{2\pi t} \qquad t \in [0, 3]
+
+with initial conditions 
+
+.. math:: x(0) = -8, \quad y(0) = 7, \quad z(0) = 27.
+
+where the parameters :math:`\sigma`, :math:`\rho`, and :math:`\beta` are to be identified from observations of the system at certain times and whose true values are 10, 15, and 8/3, respectivly. And the :math:`\sin{2\pi t}` is the exogenous input.
+
+Implementation
+--------------
+
+This description goes through the implementation of a solver for the above Lorenz system step-by-step.
+
+First, the DeepXDE, NumPy (``np``) and Sciopy (``sp``) modules are imported:
+
+.. code-block:: python
+
+    import deepxde as dde
+    import numpy as np
+    import scipy as sp
+
+We also want to define our three unknown variables, :math:`\sigma`, :math:`\rho`, and :math:`\beta` which will now be called C1, C2, and C3, respectivly. These variables are given an initial guess of 1.0.
+
+.. code-block:: python
+    
+    C1 = dde.Variable(1.0)
+    C2 = dde.Variable(1.0)
+    C3 = dde.Variable(1.0)
+
+Now we can begin by creating a ``TimeDomain`` class.
+
+.. code-block:: python
+    
+    geom = dde.geometry.TimeDomain(0, 3)
+
+Next, we define the exogenous input:
+
+.. code-block:: python
+
+    maxtime = 3
+    time = np.linspace(0, maxtime, 200)
+    ex_input = 10 * np.sin(2 * np.pi * time)
+
+Next, we can define the interpolation function of time and exogenous input:
+
+.. code-block:: python
+
+    def ex_func2(t):
+    spline = sp.interpolate.Rbf(
+        time, ex_input, function="thin_plate", smooth=0, episilon=0
+    )
+    return spline(t[:, 0:])
+
+Next, we create the Lorenz system to solve using the ``dde.grad.jacobian`` function.
+
+.. code-block:: python
+
+    def Lorenz_system(x, y, ex):
+    y1, y2, y3 = y[:, 0:1], y[:, 1:2], y[:, 2:]
+    dy1_x = dde.grad.jacobian(y, x, i=0)
+    dy2_x = dde.grad.jacobian(y, x, i=1)
+    dy3_x = dde.grad.jacobian(y, x, i=2)
+    return [
+        dy1_x - C1 * (y2 - y1),
+        dy2_x - y1 * (C2 - y3) + y2,
+        dy3_x - y1 * y2 + C3 * y3 - ex,
+    ]
+
+The first argument to ``Lorenz_system`` is the network input, i.e., the :math:`t`-coordinate. The second argument is the network output, i.e., the solution :math:`y(x,y,z)`, but here we use ``y1, y2, y3`` as the name of the coordinates x, y, and z, which correspond to the columns of datapoints in the 2D array, :math:`y`. And the third argument ``ex`` is the exogenous input.
+
+Next, we consider the initial conditions. We need to implement a function, which should return ``True`` for points inside the subdomain and ``False`` for the points outside. 
+
+.. code-block:: python
+
+    def boundary(_, on_initial):
+        return on_initial
+
+Then the initial conditions are specified using the computational domain, initial function, and boundary. The argument ``component`` refers to if this IC is for the first component (:math:`x`), the second component (:math:`y`), or the third component (:math:`z`). Note that in our case, the point :math:`t` of the initial condition is :math:`t = 0`. 
+
+.. code-block:: python
+
+    ic1 = dde.icbc.IC(geom, lambda X: x0[0], boundary, component=0)
+    ic2 = dde.icbc.IC(geom, lambda X: x0[1], boundary, component=1)
+    ic3 = dde.icbc.IC(geom, lambda X: x0[2], boundary, component=2)
+
+Then we organize and assign the train data. 
+
+.. code-block:: python
+
+    observe_t, ob_y = time, x
+    observe_y0 = dde.icbc.PointSetBC(observe_t, ob_y[:, 0:1], component=0)
+    observe_y1 = dde.icbc.PointSetBC(observe_t, ob_y[:, 1:2], component=1)
+    observe_y2 = dde.icbc.PointSetBC(observe_t, ob_y[:, 2:3], component=2)
+
+Now that the problem is fully setup, we define the PDE as: 
+
+.. code-block:: python
+
+    data = dde.data.PDE(
+        geom,
+        Lorenz_system,
+        [ic1, ic2, ic3, observe_y0, observe_y1, observe_y2],
+        num_domain=400,
+        num_boundary=2,
+        anchors=observe_t,
+        auxiliary_var_function=ex_func2,
+    )
+
+Where ``num_domain`` is the number of points inside the domain, and ``num_boundary`` is the number of points on the boundary. ``anchors`` are extra points beyond ``num_domain`` and ``num_boundary`` used for training. ``auxiliary_var_function`` is the interpolation function of time and exogenous input we defined above.
+
+Next, we choose the network. Here, we use a fully connected neural network of depth 4 (i.e., 3 hidden layers) and width 40:
+
+.. code-block:: python
+
+    net = dde.nn.FNN([1] + [40] * 3 + [3], "tanh", "Glorot uniform")
+
+Now that the PDE problem and network have been created, we build a ``Model`` and choose the optimizer, learning rate, and provide the trainable variables C1, C2, and C3:
+
+.. code-block:: python
+
+    model = dde.Model(data, net)
+    model.compile("adam", lr=0.001, external_trainable_variables=[C1, C2, C3])
+
+Next, we define the callbacks for storing results:
+
+.. code-block:: python
+
+    fnamevar = "variables.dat"
+    variable = dde.callbacks.VariableValue([C1, C2, C3], period=100, filename=fnamevar)
+
+We then train the model for 60000 iterations:
+
+.. code-block:: python
+
+    model.train(epochs=60000, callbacks=[variable])
+
+Complete code
+-------------
+
+.. literalinclude:: ../../../examples/pinn_inverse/Lorenz_inverse_forced.py
+  :language: python
+

From b3e1a9eccb2bbbe73cb82df34269a7b3f96cc2c9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 15:07:33 -0400
Subject: [PATCH 009/170] Update demo document for lorenz with exogenous input.
 inverse

---
 docs/demos/pinn_inverse.rst                                     | 2 +-
 ...nput.inverse.rst => lorenz_with exogenous_input.inverse.rst} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename docs/demos/pinn_inverse/{lorenz with exogenous input.inverse.rst => lorenz_with exogenous_input.inverse.rst} (100%)

diff --git a/docs/demos/pinn_inverse.rst b/docs/demos/pinn_inverse.rst
index 0f69dea30..790202f5a 100644
--- a/docs/demos/pinn_inverse.rst
+++ b/docs/demos/pinn_inverse.rst
@@ -10,7 +10,7 @@ ODEs
    :maxdepth: 1
 
    pinn_inverse/lorenz.inverse
-   pinn_inverse/lorenz with exogenous input.inverse
+   pinn_inverse/lorenz_with_exogenous_input.inverse
 
 
 Time-independent PDEs
diff --git a/docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst b/docs/demos/pinn_inverse/lorenz_with exogenous_input.inverse.rst
similarity index 100%
rename from docs/demos/pinn_inverse/lorenz with exogenous input.inverse.rst
rename to docs/demos/pinn_inverse/lorenz_with exogenous_input.inverse.rst

From ac810352271c3978dffb06f3bd3d00f4786fe453 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 15:20:07 -0400
Subject: [PATCH 010/170] Update demo document for lorenz with exogenous input.
 inverse

---
 ..._input.inverse.rst => lorenz_with_exogenous_input.inverse.rst} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/demos/pinn_inverse/{lorenz_with exogenous_input.inverse.rst => lorenz_with_exogenous_input.inverse.rst} (100%)

diff --git a/docs/demos/pinn_inverse/lorenz_with exogenous_input.inverse.rst b/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst
similarity index 100%
rename from docs/demos/pinn_inverse/lorenz_with exogenous_input.inverse.rst
rename to docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst

From 4141cac4151a078b76d2ee28e542958e73a680de Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 15:39:41 -0400
Subject: [PATCH 011/170] tiny change in .rst

---
 .../pinn_inverse/lorenz_with_exogenous_input.inverse.rst      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst b/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst
index 5eb66faec..37ad8baee 100644
--- a/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst
+++ b/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst
@@ -6,13 +6,13 @@ Problem setup
 
 We will solve the Lorenz system:
 
-.. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - 10\sin{2\pi t} \qquad t \in [0, 3]
+.. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - 10 \sin(2\pi t) \qquad t \in [0, 3]
 
 with initial conditions 
 
 .. math:: x(0) = -8, \quad y(0) = 7, \quad z(0) = 27.
 
-where the parameters :math:`\sigma`, :math:`\rho`, and :math:`\beta` are to be identified from observations of the system at certain times and whose true values are 10, 15, and 8/3, respectivly. And the :math:`\sin{2\pi t}` is the exogenous input.
+where the parameters :math:`\sigma`, :math:`\rho`, and :math:`\beta` are to be identified from observations of the system at certain times and whose true values are 10, 15, and 8/3, respectivly. And the :math:`\sin(2\pi t)` is the exogenous input.
 
 Implementation
 --------------

From f875020d468145e16cbc43f78708277b58dcdd64 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 16:24:41 -0400
Subject: [PATCH 012/170] change .rst name and delete one empty line in
 pinn_inverse.rst

---
 docs/demos/pinn_inverse.rst                                    | 3 +--
 ...h_exogenous_input.inverse.rst => Lorenz.inverse.forced.rst} | 0
 2 files changed, 1 insertion(+), 2 deletions(-)
 rename docs/demos/pinn_inverse/{lorenz_with_exogenous_input.inverse.rst => Lorenz.inverse.forced.rst} (100%)

diff --git a/docs/demos/pinn_inverse.rst b/docs/demos/pinn_inverse.rst
index 790202f5a..a0753145d 100644
--- a/docs/demos/pinn_inverse.rst
+++ b/docs/demos/pinn_inverse.rst
@@ -10,8 +10,7 @@ ODEs
    :maxdepth: 1
 
    pinn_inverse/lorenz.inverse
-   pinn_inverse/lorenz_with_exogenous_input.inverse
-
+   pinn_inverse/Lorenz_inverse_forced
 
 Time-independent PDEs
 ---------------------
diff --git a/docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst b/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
similarity index 100%
rename from docs/demos/pinn_inverse/lorenz_with_exogenous_input.inverse.rst
rename to docs/demos/pinn_inverse/Lorenz.inverse.forced.rst

From 46a8df16fc3b4bd94bc393dfe298aded55607520 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 16:44:20 -0400
Subject: [PATCH 013/170] tiny change in pinn_inverse.rst

---
 docs/demos/pinn_inverse.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demos/pinn_inverse.rst b/docs/demos/pinn_inverse.rst
index a0753145d..544b5556f 100644
--- a/docs/demos/pinn_inverse.rst
+++ b/docs/demos/pinn_inverse.rst
@@ -10,7 +10,7 @@ ODEs
    :maxdepth: 1
 
    pinn_inverse/lorenz.inverse
-   pinn_inverse/Lorenz_inverse_forced
+   pinn_inverse/Lorenz.inverse.forced
 
 Time-independent PDEs
 ---------------------

From c06657da10fd9a74568b3788d54297dd236d9332 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 16:53:10 -0400
Subject: [PATCH 014/170] tiny change in Lorenz.inverse.forced.rst

---
 docs/demos/pinn_inverse/Lorenz.inverse.forced.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst b/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
index 37ad8baee..eaeb11768 100644
--- a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
+++ b/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
@@ -19,7 +19,7 @@ Implementation
 
 This description goes through the implementation of a solver for the above Lorenz system step-by-step.
 
-First, the DeepXDE, NumPy (``np``) and Sciopy (``sp``) modules are imported:
+First, the DeepXDE, NumPy (``np``) and Scipy (``sp``) modules are imported:
 
 .. code-block:: python
 

From d5da9f75a45bc530bb18da514f16e6b606507e17 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 17:04:59 -0400
Subject: [PATCH 015/170] fix typo in Lorenz.inverse.forced.rst

---
 docs/demos/pinn_inverse/Lorenz.inverse.forced.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst b/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
index eaeb11768..bd69d8f47 100644
--- a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
+++ b/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
@@ -19,7 +19,7 @@ Implementation
 
 This description goes through the implementation of a solver for the above Lorenz system step-by-step.
 
-First, the DeepXDE, NumPy (``np``) and Scipy (``sp``) modules are imported:
+First, the DeepXDE, Numpy (``np``) and Scipy (``sp``) modules are imported:
 
 .. code-block:: python
 

From c1bcb16aac84a3442de3ed377ff6b42961cc571b Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 17:48:48 -0400
Subject: [PATCH 016/170] fix typo in pinn_inverse.rst and rename
 lorenz.inverse.forced.rst

---
 docs/demos/pinn_inverse.rst                   |  2 +-
 ...e.forced.rst => lorenz.inverse.forced.rst} | 28 +++++++++----------
 2 files changed, 14 insertions(+), 16 deletions(-)
 rename docs/demos/pinn_inverse/{Lorenz.inverse.forced.rst => lorenz.inverse.forced.rst} (91%)

diff --git a/docs/demos/pinn_inverse.rst b/docs/demos/pinn_inverse.rst
index 544b5556f..d0fdbcf91 100644
--- a/docs/demos/pinn_inverse.rst
+++ b/docs/demos/pinn_inverse.rst
@@ -10,7 +10,7 @@ ODEs
    :maxdepth: 1
 
    pinn_inverse/lorenz.inverse
-   pinn_inverse/Lorenz.inverse.forced
+   pinn_inverse/lorenz.inverse.forced
 
 Time-independent PDEs
 ---------------------
diff --git a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
similarity index 91%
rename from docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
rename to docs/demos/pinn_inverse/lorenz.inverse.forced.rst
index bd69d8f47..bce359c76 100644
--- a/docs/demos/pinn_inverse/Lorenz.inverse.forced.rst
+++ b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
@@ -53,26 +53,24 @@ Next, we can define the interpolation function of time and exogenous input:
 
 .. code-block:: python
 
-    def ex_func2(t):
-    spline = sp.interpolate.Rbf(
-        time, ex_input, function="thin_plate", smooth=0, episilon=0
-    )
-    return spline(t[:, 0:])
+    def ex_func(t):
+        spline = sp.interpolate.Rbf(
+            time, ex_input, function="thin_plate", smooth=0, episilon=0
+        )
+        return spline(t)
 
 Next, we create the Lorenz system to solve using the ``dde.grad.jacobian`` function.
 
 .. code-block:: python
 
-    def Lorenz_system(x, y, ex):
-    y1, y2, y3 = y[:, 0:1], y[:, 1:2], y[:, 2:]
-    dy1_x = dde.grad.jacobian(y, x, i=0)
-    dy2_x = dde.grad.jacobian(y, x, i=1)
-    dy3_x = dde.grad.jacobian(y, x, i=2)
-    return [
-        dy1_x - C1 * (y2 - y1),
-        dy2_x - y1 * (C2 - y3) + y2,
-        dy3_x - y1 * y2 + C3 * y3 - ex,
-    ]
+    def LorezODE(x, t):
+        x1, x2, x3 = x
+        dxdt = [
+            C1true * (x2 - x1),
+            x1 * (C2true - x3) - x2,
+            x1 * x2 - C3true * x3 + ex_func(t),
+        ]
+        return dxdt
 
 The first argument to ``Lorenz_system`` is the network input, i.e., the :math:`t`-coordinate. The second argument is the network output, i.e., the solution :math:`y(x,y,z)`, but here we use ``y1, y2, y3`` as the name of the coordinates x, y, and z, which correspond to the columns of datapoints in the 2D array, :math:`y`. And the third argument ``ex`` is the exogenous input.
 

From d89816fde3aa83625947fab9ca773c5af143630e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 18:09:55 -0400
Subject: [PATCH 017/170] fix typo lorenz.inverse.forced.rst

---
 .../pinn_inverse/lorenz.inverse.forced.rst    | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/docs/demos/pinn_inverse/lorenz.inverse.forced.rst b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
index bce359c76..843896e27 100644
--- a/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
+++ b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
@@ -6,7 +6,9 @@ Problem setup
 
 We will solve the Lorenz system:
 
-.. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - 10 \sin(2\pi t) \qquad t \in [0, 3]
+.. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - f(t) \qquad t \in [0, 3]
+
+Where :math:`f(t)=10\sin(2\pi t)` is the exgoenous input.
 
 with initial conditions 
 
@@ -41,7 +43,7 @@ Now we can begin by creating a ``TimeDomain`` class.
     
     geom = dde.geometry.TimeDomain(0, 3)
 
-Next, we define the exogenous input:
+Next, we define the exogenous input :math:`f(t)`:
 
 .. code-block:: python
 
@@ -53,24 +55,26 @@ Next, we can define the interpolation function of time and exogenous input:
 
 .. code-block:: python
 
-    def ex_func(t):
+    def ex_func2(t):
         spline = sp.interpolate.Rbf(
             time, ex_input, function="thin_plate", smooth=0, episilon=0
         )
-        return spline(t)
+        return spline(t[:, 0:])
 
 Next, we create the Lorenz system to solve using the ``dde.grad.jacobian`` function.
 
 .. code-block:: python
 
-    def LorezODE(x, t):
-        x1, x2, x3 = x
-        dxdt = [
-            C1true * (x2 - x1),
-            x1 * (C2true - x3) - x2,
-            x1 * x2 - C3true * x3 + ex_func(t),
+    def Lorenz_system(x, y, ex):
+        y1, y2, y3 = y[:, 0:1], y[:, 1:2], y[:, 2:]
+        dy1_x = dde.grad.jacobian(y, x, i=0)
+        dy2_x = dde.grad.jacobian(y, x, i=1)
+        dy3_x = dde.grad.jacobian(y, x, i=2)
+        return [
+            dy1_x - C1 * (y2 - y1),
+            dy2_x - y1 * (C2 - y3) + y2,
+            dy3_x - y1 * y2 + C3 * y3 - ex,
         ]
-        return dxdt
 
 The first argument to ``Lorenz_system`` is the network input, i.e., the :math:`t`-coordinate. The second argument is the network output, i.e., the solution :math:`y(x,y,z)`, but here we use ``y1, y2, y3`` as the name of the coordinates x, y, and z, which correspond to the columns of datapoints in the 2D array, :math:`y`. And the third argument ``ex`` is the exogenous input.
 

From 8cee95559e5b4916035687df708058705879d353 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 May 2022 19:14:42 -0400
Subject: [PATCH 018/170] some tiny changes in lorenz.inverse.forced.rst

---
 docs/demos/pinn_inverse/lorenz.inverse.forced.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/demos/pinn_inverse/lorenz.inverse.forced.rst b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
index 843896e27..9e9f9e506 100644
--- a/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
+++ b/docs/demos/pinn_inverse/lorenz.inverse.forced.rst
@@ -8,13 +8,13 @@ We will solve the Lorenz system:
 
 .. math:: \frac{dx}{dt} = \sigma(y-x), \quad \frac{dy}{dt} = x (\rho - z) - y, \quad \frac{dz}{dt} = x y - \beta z - f(t) \qquad t \in [0, 3]
 
-Where :math:`f(t)=10\sin(2\pi t)` is the exgoenous input.
+where :math:`f(t)=10\sin(2\pi t)` is the exgoenous input.
 
-with initial conditions 
+The initial condition is:
 
 .. math:: x(0) = -8, \quad y(0) = 7, \quad z(0) = 27.
 
-where the parameters :math:`\sigma`, :math:`\rho`, and :math:`\beta` are to be identified from observations of the system at certain times and whose true values are 10, 15, and 8/3, respectivly. And the :math:`\sin(2\pi t)` is the exogenous input.
+where the parameters :math:`\sigma`, :math:`\rho`, and :math:`\beta` are to be identified from observations of the system at certain times and whose true values are 10, 15, and 8/3, respectivly. 
 
 Implementation
 --------------
@@ -43,7 +43,7 @@ Now we can begin by creating a ``TimeDomain`` class.
     
     geom = dde.geometry.TimeDomain(0, 3)
 
-Next, we define the exogenous input :math:`f(t)`:
+Next, we assume that we don't know the formula of :math:`f(t)`, and we only know :math:`f(t)` at 200 points.
 
 .. code-block:: python
 
@@ -51,7 +51,7 @@ Next, we define the exogenous input :math:`f(t)`:
     time = np.linspace(0, maxtime, 200)
     ex_input = 10 * np.sin(2 * np.pi * time)
 
-Next, we can define the interpolation function of time and exogenous input:
+Next, we can define an interpolation function of :math:`f(t)` for any t:
 
 .. code-block:: python
 
@@ -116,7 +116,7 @@ Now that the problem is fully setup, we define the PDE as:
         auxiliary_var_function=ex_func2,
     )
 
-Where ``num_domain`` is the number of points inside the domain, and ``num_boundary`` is the number of points on the boundary. ``anchors`` are extra points beyond ``num_domain`` and ``num_boundary`` used for training. ``auxiliary_var_function`` is the interpolation function of time and exogenous input we defined above.
+Where ``num_domain`` is the number of points inside the domain, and ``num_boundary`` is the number of points on the boundary. ``anchors`` are extra points beyond ``num_domain`` and ``num_boundary`` used for training. ``auxiliary_var_function`` is the interpolation function of :math:`f(t)` we defined above.
 
 Next, we choose the network. Here, we use a fully connected neural network of depth 4 (i.e., 3 hidden layers) and width 40:
 

From c4e5b487323b28c14974cca86c979bb984e542e3 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 2 Jun 2022 19:33:37 -0400
Subject: [PATCH 019/170] add deeponet implemented by pytorch

---
 deepxde/model.py               |  11 ++-
 deepxde/nn/pytorch/__init__.py |   3 +-
 deepxde/nn/pytorch/deeponet.py | 138 +++++++++++++++++++++++++++++++++
 3 files changed, 149 insertions(+), 3 deletions(-)
 create mode 100644 deepxde/nn/pytorch/deeponet.py

diff --git a/deepxde/model.py b/deepxde/model.py
index eb52facab..53e9a26d1 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -246,8 +246,15 @@ def outputs(training, inputs):
 
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
-            inputs = torch.as_tensor(inputs)
-            inputs.requires_grad_()
+
+            if isinstance(inputs,tuple):
+                multi_inputs = torch.as_tensor(inputs[0])
+                inputs_num = len(inputs)
+                for i in range(1,inputs_num):
+                    inputs_i = torch.as_tensor(inputs[i])
+                    inputs_i.requires_grad_()
+                    multi_inputs = (multi_inputs, inputs_i)
+                inputs = multi_inputs
             outputs_ = self.net(inputs)
             # Data losses
             if targets is not None:
diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index e5942c784..5dcbb8396 100644
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -1,6 +1,7 @@
 """Package for pytorch NN modules."""
 
-__all__ = ["FNN", "NN", "PFNN"]
+__all__ = ["FNN", "NN", "PFNN", "DeepONetCartesianProd", "PODDeepONet"]
 
 from .fnn import FNN, PFNN
 from .nn import NN
+from .deeponet import DeepONetCartesianProd, PODDeepONet
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
new file mode 100644
index 000000000..f8266000e
--- /dev/null
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -0,0 +1,138 @@
+from .fnn import FNN
+from .nn import NN
+from .. import activations
+import torch
+
+class DeepONetCartesianProd(NN):
+    """Deep operator network for dataset in the format of Cartesian product.
+
+    Args:
+        layer_sizes_branch: A list of integers as the width of a fully connected network,
+            or `(dim, f)` where `dim` is the input dimension and `f` is a network
+            function. The width of the last layer in the branch and trunk net should be
+            equal.
+        layer_sizes_trunk (list): A list of integers as the width of a fully connected
+            network.
+        activation: If `activation` is a ``string``, then the same activation is used in
+            both trunk and branch nets. If `activation` is a ``dict``, then the trunk
+            net uses the activation `activation["trunk"]`, and the branch net uses
+            `activation["branch"]`.
+    """
+
+    def __init__(
+        self,
+        layer_sizes_branch,
+        layer_sizes_trunk,
+        activation,
+        kernel_initializer,
+    ):
+        super().__init__()
+        if isinstance(activation, dict):
+            activation_branch = activation["branch"]
+            self.activation_trunk = activations.get(activation["trunk"])
+        else:
+            activation_branch = self.activation_trunk = activations.get(activation)
+        if callable(layer_sizes_branch[1]):
+            # User-defined network
+            self.branch = layer_sizes_branch[1]
+        else:
+            # Fully connected network
+            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
+        self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
+        #self.b = tf.Variable(tf.zeros(1))
+        self.b = torch.tensor(0.0, requires_grad=True)
+
+    def forward(self, inputs, training=False):
+        x_func = inputs[0]
+        x_loc = inputs[1]
+        # Branch net to encode the input function
+        x_func = self.branch(x_func)
+        # Trunk net to encode the domain of the output function
+        if self._input_transform is not None:
+            x_loc = self._input_transform(x_loc)
+        x_loc = self.activation_trunk(self.trunk(x_loc))
+        # Dot product
+        if x_func.shape[-1] != x_loc.shape[-1]:
+            raise AssertionError(
+                "Output sizes of branch net and trunk net do not match."
+            )
+        x = torch.einsum("bi,ni->bn", x_func, x_loc)
+        # Add bias
+        x += self.b
+
+        if self._output_transform is not None:
+            x = self._output_transform(inputs, x)
+        return x
+
+
+class PODDeepONet(NN):
+    """Deep operator network with proper orthogonal decomposition (POD) for dataset in
+    the format of Cartesian product.
+
+    Args:
+        pod_basis: POD basis used in the trunk net.
+        layer_sizes_branch: A list of integers as the width of a fully connected network,
+            or `(dim, f)` where `dim` is the input dimension and `f` is a network
+            function. The width of the last layer in the branch and trunk net should be
+            equal.
+        activation: If `activation` is a ``string``, then the same activation is used in
+            both trunk and branch nets. If `activation` is a ``dict``, then the trunk
+            net uses the activation `activation["trunk"]`, and the branch net uses
+            `activation["branch"]`.
+        layer_sizes_trunk (list): A list of integers as the width of a fully connected
+            network. If ``None``, then only use POD basis as the trunk net.
+
+    References:
+        `L. Lu, X. Meng, S. Cai, Z. Mao, S. Goswami, Z. Zhang, & G. E. Karniadakis. A
+        comprehensive and fair comparison of two neural operators (with practical
+        extensions) based on FAIR data. arXiv preprint arXiv:2111.05512, 2021
+        <https://arxiv.org/abs/2111.05512>`_.
+    """
+    def __init__(
+        self,
+        pod_basis,
+        layer_sizes_branch,
+        activation,
+        kernel_initializer,
+        layer_sizes_trunk=None,
+    ):
+        super().__init__()
+        self.pod_basis = torch.tensor(pod_basis, dtype=torch.float32, requires_grad=True)
+        if isinstance(activation, dict):
+            activation_branch = activation["branch"]
+            self.activation_trunk = activations.get(activation["trunk"])
+        else:
+            activation_branch = self.activation_trunk = activations.get(activation)
+
+        if callable(layer_sizes_branch[1]):
+            # User-defined network
+            self.branch = layer_sizes_branch[1]
+        else:
+            # Fully connected network
+            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
+
+        self.trunk = None
+        if layer_sizes_trunk is not None:
+            self.trunk = FNN(
+                layer_sizes_trunk, self.activation_trunk, kernel_initializer
+            )
+            self.b = torch.tensor(0.0, requires_grad=True)
+
+    def forward(self, inputs, training=False):
+        x_func = inputs[0]
+        x_loc = inputs[1]
+
+        # Branch net to encode the input function
+        x_func = self.branch(x_func)
+        # Trunk net to encode the domain of the output function
+        if self.trunk is None:
+            # POD only
+            x = torch.einsum("bi,ni->bn", x_func, self.pod_basis)
+        else:
+            x_loc = self.activation_trunk(self.trunk(x_loc))
+            x = torch.einsum("bi,ni->bn", x_func, torch.cat((self.pod_basis, x_loc), 1))
+            x += self.b
+
+        if self._output_transform is not None:
+            x = self._output_transform(inputs, x)
+        return x

From 86e8f2f782fe55bfa2166fda2213c0bc61ea7cd2 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 2 Jun 2022 19:49:07 -0400
Subject: [PATCH 020/170] add deeponet implemented by pytorch

---
 deepxde/model.py               | 4 +++-
 deepxde/nn/pytorch/__init__.py | 0
 deepxde/nn/pytorch/deeponet.py | 0
 3 files changed, 3 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 deepxde/nn/pytorch/__init__.py
 mode change 100644 => 100755 deepxde/nn/pytorch/deeponet.py

diff --git a/deepxde/model.py b/deepxde/model.py
index 4771428d0..ffd648bf5 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -255,6 +255,8 @@ def outputs_losses(training, inputs, targets, losses_fn):
                     inputs_i.requires_grad_()
                     multi_inputs = (multi_inputs, inputs_i)
                 inputs = multi_inputs
+            #inputs = torch.as_tensor(inputs)
+            #inputs.requires_grad_()
             outputs_ = self.net(inputs)
             # Data losses
             if targets is not None:
@@ -419,7 +421,7 @@ def _outputs(self, training, inputs):
         if backend_name == "tensorflow.compat.v1":
             feed_dict = self.net.feed_dict(training, inputs)
             return self.sess.run(self.outputs, feed_dict=feed_dict)
-        if backend_name in ["tensorflow", "pytorch", "paddle"]:
+        if backend_name in ["tensorflow", "pytorch"]:
             outs = self.outputs(training, inputs)
         elif backend_name == "jax":
             outs = self.outputs(self.net.params, training, inputs)
diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
old mode 100644
new mode 100755
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
old mode 100644
new mode 100755

From 31932a45a9926c1304b3b77af03cb56cf4e0a87f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 01:09:22 -0400
Subject: [PATCH 021/170] tiny changes in deeponet.py and model.py

---
 deepxde/model.py               | 10 +++++-----
 deepxde/nn/pytorch/deeponet.py | 16 +++++++++-------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index ffd648bf5..cabbbf395 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -246,17 +246,17 @@ def outputs(training, inputs):
 
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
-
-            if isinstance(inputs,tuple):
+            if isinstance(inputs, tuple):
                 multi_inputs = torch.as_tensor(inputs[0])
                 inputs_num = len(inputs)
-                for i in range(1,inputs_num):
+                for i in range(1, inputs_num):
                     inputs_i = torch.as_tensor(inputs[i])
                     inputs_i.requires_grad_()
                     multi_inputs = (multi_inputs, inputs_i)
                 inputs = multi_inputs
-            #inputs = torch.as_tensor(inputs)
-            #inputs.requires_grad_()
+            else:
+                inputs = torch.as_tensor(inputs)
+                inputs.requires_grad_()
             outputs_ = self.net(inputs)
             # Data losses
             if targets is not None:
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index f8266000e..b1d8dfed7 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -3,6 +3,7 @@
 from .. import activations
 import torch
 
+
 class DeepONetCartesianProd(NN):
     """Deep operator network for dataset in the format of Cartesian product.
 
@@ -39,7 +40,7 @@ def __init__(
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        #self.b = tf.Variable(tf.zeros(1))
+        # self.b = tf.Variable(tf.zeros(1))
         self.b = torch.tensor(0.0, requires_grad=True)
 
     def forward(self, inputs, training=False):
@@ -88,29 +89,33 @@ class PODDeepONet(NN):
         extensions) based on FAIR data. arXiv preprint arXiv:2111.05512, 2021
         <https://arxiv.org/abs/2111.05512>`_.
     """
+
+
+class PODDeepONet(NN):
     def __init__(
         self,
         pod_basis,
         layer_sizes_branch,
+        layer_sizes_trunk,
         activation,
         kernel_initializer,
-        layer_sizes_trunk=None,
     ):
         super().__init__()
-        self.pod_basis = torch.tensor(pod_basis, dtype=torch.float32, requires_grad=True)
         if isinstance(activation, dict):
             activation_branch = activation["branch"]
             self.activation_trunk = activations.get(activation["trunk"])
         else:
             activation_branch = self.activation_trunk = activations.get(activation)
 
+        self.pod_basis = torch.tensor(
+            pod_basis, dtype=torch.float32, requires_grad=True
+        )
         if callable(layer_sizes_branch[1]):
             # User-defined network
             self.branch = layer_sizes_branch[1]
         else:
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
-
         self.trunk = None
         if layer_sizes_trunk is not None:
             self.trunk = FNN(
@@ -122,9 +127,7 @@ def forward(self, inputs, training=False):
         x_func = inputs[0]
         x_loc = inputs[1]
 
-        # Branch net to encode the input function
         x_func = self.branch(x_func)
-        # Trunk net to encode the domain of the output function
         if self.trunk is None:
             # POD only
             x = torch.einsum("bi,ni->bn", x_func, self.pod_basis)
@@ -132,7 +135,6 @@ def forward(self, inputs, training=False):
             x_loc = self.activation_trunk(self.trunk(x_loc))
             x = torch.einsum("bi,ni->bn", x_func, torch.cat((self.pod_basis, x_loc), 1))
             x += self.b
-
         if self._output_transform is not None:
             x = self._output_transform(inputs, x)
         return x

From 4ca1ef01646ebfff0a9a4874efe757d46777a01c Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 01:15:59 -0400
Subject: [PATCH 022/170] tiny changes in deeponet.py

---
 deepxde/nn/pytorch/deeponet.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index b1d8dfed7..637dc1543 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -90,8 +90,6 @@ class PODDeepONet(NN):
         <https://arxiv.org/abs/2111.05512>`_.
     """
 
-
-class PODDeepONet(NN):
     def __init__(
         self,
         pod_basis,

From 42a80f97ec816b7bb4915b9127ed985482a350af Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 13:08:08 -0400
Subject: [PATCH 023/170] tiny changes in deeponet.py, model.py and
 __init__.py; and add "Step" method of Lr decay in Pytorch

---
 deepxde/model.py                         |  5 ++-
 deepxde/nn/pytorch/__init__.py           |  5 +--
 deepxde/nn/pytorch/deeponet.py           | 11 ++++---
 deepxde/optimizers/pytorch/optimizers.py | 41 +++++++++++++++++++-----
 4 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index cabbbf395..6a6d70629 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -39,6 +39,7 @@ def __init__(self, data, net):
 
         # Backend-dependent attributes
         self.opt = None
+        self.lr_scheduler = None
         # Tensor or callable
         self.outputs = None
         self.outputs_losses_train = None
@@ -285,7 +286,7 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        self.opt = optimizers.get(
+        self.opt, self.lr_scheduler = optimizers.get(
             trainable_variables, self.opt_name, learning_rate=lr, decay=decay
         )
 
@@ -298,6 +299,8 @@ def closure():
                 return total_loss
 
             self.opt.step(closure)
+            if self.lr_scheduler is not None:
+                self.lr_scheduler.step()
 
         # Callables
         self.outputs = outputs
diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index 5dcbb8396..99bd141a2 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -1,7 +1,8 @@
 """Package for pytorch NN modules."""
 
-__all__ = ["FNN", "NN", "PFNN", "DeepONetCartesianProd", "PODDeepONet"]
+__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet"]
 
+from .deeponet import DeepONetCartesianProd, PODDeepONet
 from .fnn import FNN, PFNN
 from .nn import NN
-from .deeponet import DeepONetCartesianProd, PODDeepONet
+
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 637dc1543..d86dd3d88 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -1,7 +1,8 @@
+import torch
+
 from .fnn import FNN
 from .nn import NN
 from .. import activations
-import torch
 
 
 class DeepONetCartesianProd(NN):
@@ -40,7 +41,6 @@ def __init__(
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        # self.b = tf.Variable(tf.zeros(1))
         self.b = torch.tensor(0.0, requires_grad=True)
 
     def forward(self, inputs, training=False):
@@ -94,9 +94,9 @@ def __init__(
         self,
         pod_basis,
         layer_sizes_branch,
-        layer_sizes_trunk,
         activation,
         kernel_initializer,
+        layer_sizes_trunk=None,
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -105,9 +105,10 @@ def __init__(
         else:
             activation_branch = self.activation_trunk = activations.get(activation)
 
-        self.pod_basis = torch.tensor(
-            pod_basis, dtype=torch.float32, requires_grad=True
+        self.pod_basis = torch.as_tensor(
+            pod_basis, dtype=torch.float32
         )
+        self.pod_basis.requires_grad_()
         if callable(layer_sizes_branch[1]):
             # User-defined network
             self.branch = layer_sizes_branch[1]
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index a96fa3339..07ff4f3cc 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -11,13 +11,15 @@ def is_external_optimizer(optimizer):
 
 def get(params, optimizer, learning_rate=None, decay=None):
     """Retrieves an Optimizer instance."""
+
+    #Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
-        return optimizer
+        optim =  optimizer
 
     if optimizer in ["L-BFGS", "L-BFGS-B"]:
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
-        return torch.optim.LBFGS(
+        optim =  torch.optim.LBFGS(
             params,
             lr=1,
             max_iter=LBFGS_options["iter_per_step"],
@@ -31,11 +33,34 @@ def get(params, optimizer, learning_rate=None, decay=None):
     if learning_rate is None:
         raise ValueError("No learning rate for {}.".format(optimizer))
 
+    # Default optimizer is Adam
+    optim = torch.optim.Adam(params, lr=learning_rate)
+
+    if optimizer == "SGD":
+        optim = torch.optim.SGD(params, lr = learning_rate)
+
+    if optimizer == "RMSprop":
+        optim = torch.optim.RMSprop(params,lr=learning_rate)
+
+    if optimizer not in ["SGD","RMSprop","adam"]:
+        raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
+
     if decay is not None:
-        # TODO: learning rate decay
+
+        lr_scheduler = _get_learningrate_scheduler(optim, decay)
+    else:
+        lr_scheduler = None
+
+    return optim, lr_scheduler
+
+def _get_learningrate_scheduler(optim, decay):
+    if decay is None:
+        return None
+
+    if decay[0] == "Step":
+        return torch.optim.lr_scheduler.StepLR(optim, step_size=decay[1], gamma=decay[2])
+    else:
+        # TODO: More learning rate scheduler
         raise NotImplementedError(
-            "learning rate decay to be implemented for backend pytorch."
-        )
-    if optimizer == "adam":
-        return torch.optim.Adam(params, lr=learning_rate)
-    raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
+            f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
+        )
\ No newline at end of file

From 8b2b4494a9c37261740cb3486fe18a59804aaf4e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 13:52:41 -0400
Subject: [PATCH 024/170] Fix problem in model.py

---
 deepxde/model.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 6a6d70629..afb954997 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -248,12 +248,12 @@ def outputs(training, inputs):
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
             if isinstance(inputs, tuple):
-                multi_inputs = torch.as_tensor(inputs[0])
+                multi_inputs = [torch.as_tensor(inputs[0])]
                 inputs_num = len(inputs)
                 for i in range(1, inputs_num):
                     inputs_i = torch.as_tensor(inputs[i])
                     inputs_i.requires_grad_()
-                    multi_inputs = (multi_inputs, inputs_i)
+                    multi_inputs.append(inputs_i)
                 inputs = multi_inputs
             else:
                 inputs = torch.as_tensor(inputs)

From 68b90c0911b8ab033dfa98b13f950b4992f24513 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 14:10:56 -0400
Subject: [PATCH 025/170] roll back to no lr decay

---
 deepxde/optimizers/pytorch/optimizers.py | 41 +++++-------------------
 1 file changed, 8 insertions(+), 33 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 07ff4f3cc..627df134b 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -11,15 +11,13 @@ def is_external_optimizer(optimizer):
 
 def get(params, optimizer, learning_rate=None, decay=None):
     """Retrieves an Optimizer instance."""
-
-    #Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
-        optim =  optimizer
+        return optimizer
 
     if optimizer in ["L-BFGS", "L-BFGS-B"]:
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
-        optim =  torch.optim.LBFGS(
+        return torch.optim.LBFGS(
             params,
             lr=1,
             max_iter=LBFGS_options["iter_per_step"],
@@ -33,34 +31,11 @@ def get(params, optimizer, learning_rate=None, decay=None):
     if learning_rate is None:
         raise ValueError("No learning rate for {}.".format(optimizer))
 
-    # Default optimizer is Adam
-    optim = torch.optim.Adam(params, lr=learning_rate)
-
-    if optimizer == "SGD":
-        optim = torch.optim.SGD(params, lr = learning_rate)
-
-    if optimizer == "RMSprop":
-        optim = torch.optim.RMSprop(params,lr=learning_rate)
-
-    if optimizer not in ["SGD","RMSprop","adam"]:
-        raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
-
     if decay is not None:
-
-        lr_scheduler = _get_learningrate_scheduler(optim, decay)
-    else:
-        lr_scheduler = None
-
-    return optim, lr_scheduler
-
-def _get_learningrate_scheduler(optim, decay):
-    if decay is None:
-        return None
-
-    if decay[0] == "Step":
-        return torch.optim.lr_scheduler.StepLR(optim, step_size=decay[1], gamma=decay[2])
-    else:
-        # TODO: More learning rate scheduler
+        # TODO: learning rate decay
         raise NotImplementedError(
-            f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
-        )
\ No newline at end of file
+            "learning rate decay to be implemented for backend pytorch."
+        )
+    if optimizer == "adam":
+        return torch.optim.Adam(params, lr=learning_rate)
+    raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
\ No newline at end of file

From c90661cc0507537db327d3254ae5130f4eeccfc4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 14:32:06 -0400
Subject: [PATCH 026/170] roll back model.py to no lr decay

---
 deepxde/model.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index afb954997..14dcbcbe9 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -286,7 +286,7 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        self.opt, self.lr_scheduler = optimizers.get(
+        self.opt= optimizers.get(
             trainable_variables, self.opt_name, learning_rate=lr, decay=decay
         )
 
@@ -299,8 +299,6 @@ def closure():
                 return total_loss
 
             self.opt.step(closure)
-            if self.lr_scheduler is not None:
-                self.lr_scheduler.step()
 
         # Callables
         self.outputs = outputs

From e2370b75932e23d27675d2952723a8bc682a5484 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 15:10:06 -0400
Subject: [PATCH 027/170] roll back model.py

---
 deepxde/model.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 14dcbcbe9..14c6da30c 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -39,7 +39,6 @@ def __init__(self, data, net):
 
         # Backend-dependent attributes
         self.opt = None
-        self.lr_scheduler = None
         # Tensor or callable
         self.outputs = None
         self.outputs_losses_train = None

From 03ed52b89ce2323fb097cfc2fb923ed9235af94e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 15:15:37 -0400
Subject: [PATCH 028/170] reformat model.py and pytorch/optimizers.py via black

---
 deepxde/model.py                         | 2 +-
 deepxde/optimizers/pytorch/optimizers.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 14c6da30c..36cdcc68f 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -285,7 +285,7 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        self.opt= optimizers.get(
+        self.opt = optimizers.get(
             trainable_variables, self.opt_name, learning_rate=lr, decay=decay
         )
 
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 627df134b..a96fa3339 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -38,4 +38,4 @@ def get(params, optimizer, learning_rate=None, decay=None):
         )
     if optimizer == "adam":
         return torch.optim.Adam(params, lr=learning_rate)
-    raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
\ No newline at end of file
+    raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")

From 501b74c061fc248cc5c3404a223c2b4b1be93b8f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 16:05:36 -0400
Subject: [PATCH 029/170] Fix problems

---
 deepxde/model.py               | 4 ++--
 deepxde/nn/pytorch/deeponet.py | 9 +++------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 36cdcc68f..f69ba586d 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -247,7 +247,7 @@ def outputs(training, inputs):
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
             if isinstance(inputs, tuple):
-                multi_inputs = [torch.as_tensor(inputs[0])]
+                multi_inputs = [torch.as_tensor(inputs[0]).requires_grad_()]
                 inputs_num = len(inputs)
                 for i in range(1, inputs_num):
                     inputs_i = torch.as_tensor(inputs[i])
@@ -421,7 +421,7 @@ def _outputs(self, training, inputs):
         if backend_name == "tensorflow.compat.v1":
             feed_dict = self.net.feed_dict(training, inputs)
             return self.sess.run(self.outputs, feed_dict=feed_dict)
-        if backend_name in ["tensorflow", "pytorch"]:
+        if backend_name in ["tensorflow", "pytorch", "paddle"]:
             outs = self.outputs(training, inputs)
         elif backend_name == "jax":
             outs = self.outputs(self.net.params, training, inputs)
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index d86dd3d88..fb20d29c5 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -43,7 +43,7 @@ def __init__(
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
 
-    def forward(self, inputs, training=False):
+    def forward(self, inputs):
         x_func = inputs[0]
         x_loc = inputs[1]
         # Branch net to encode the input function
@@ -105,10 +105,7 @@ def __init__(
         else:
             activation_branch = self.activation_trunk = activations.get(activation)
 
-        self.pod_basis = torch.as_tensor(
-            pod_basis, dtype=torch.float32
-        )
-        self.pod_basis.requires_grad_()
+        self.pod_basis = torch.as_tensor(pod_basis, dtype=torch.float32)
         if callable(layer_sizes_branch[1]):
             # User-defined network
             self.branch = layer_sizes_branch[1]
@@ -122,7 +119,7 @@ def __init__(
             )
             self.b = torch.tensor(0.0, requires_grad=True)
 
-    def forward(self, inputs, training=False):
+    def forward(self, inputs):
         x_func = inputs[0]
         x_loc = inputs[1]
 

From 681031987f3727f56e781fdfd26f940b83fcd3f7 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 16:26:09 -0400
Subject: [PATCH 030/170] more proper way in model.py

---
 deepxde/model.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index f69ba586d..7288cad95 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -247,13 +247,7 @@ def outputs(training, inputs):
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
             if isinstance(inputs, tuple):
-                multi_inputs = [torch.as_tensor(inputs[0]).requires_grad_()]
-                inputs_num = len(inputs)
-                for i in range(1, inputs_num):
-                    inputs_i = torch.as_tensor(inputs[i])
-                    inputs_i.requires_grad_()
-                    multi_inputs.append(inputs_i)
-                inputs = multi_inputs
+                inputs = tuple(map(lambda x: torch.as_tensor(x).requires_grad_(), inputs))
             else:
                 inputs = torch.as_tensor(inputs)
                 inputs.requires_grad_()

From 4f5e2a6013e870cf10319dbbc2da9524e011b173 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 17:34:53 -0400
Subject: [PATCH 031/170] update model.py

---
 deepxde/model.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 7288cad95..c0861f7ce 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -242,7 +242,11 @@ def _compile_pytorch(self, lr, loss_fn, decay, loss_weights):
         def outputs(training, inputs):
             self.net.train(mode=training)
             with torch.no_grad():
-                return self.net(torch.as_tensor(inputs))
+                if isinstance(inputs, tuple):
+                    inputs = tuple(map(lambda x: torch.as_tensor(x), inputs))
+                else:
+                    inputs = torch.as_tensor(inputs)
+                return self.net(inputs)
 
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)

From 9bdaf81fc832b87e2f2f8799b2b7e8722d0231d2 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 17:43:16 -0400
Subject: [PATCH 032/170] delete trailing newlines

---
 deepxde/nn/pytorch/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index 99bd141a2..bebdc53d6 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -5,4 +5,3 @@
 from .deeponet import DeepONetCartesianProd, PODDeepONet
 from .fnn import FNN, PFNN
 from .nn import NN
-

From 5fb0228af8cb4a27cdb5e9a03a30a40f143c82eb Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 18:10:08 -0400
Subject: [PATCH 033/170] tiny change in model.py

---
 deepxde/model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index c0861f7ce..a33f85970 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -243,7 +243,7 @@ def outputs(training, inputs):
             self.net.train(mode=training)
             with torch.no_grad():
                 if isinstance(inputs, tuple):
-                    inputs = tuple(map(lambda x: torch.as_tensor(x), inputs))
+                    inputs = tuple(map(torch.as_tensor, inputs))
                 else:
                     inputs = torch.as_tensor(inputs)
                 return self.net(inputs)

From a23d0a7688024976015ca94ef83c83689ca16208 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 18:28:33 -0400
Subject: [PATCH 034/170] modify pytorch/optimizers.py to add "Step" lr decay
 method

---
 deepxde/model.py                         |  5 ++-
 deepxde/optimizers/pytorch/optimizers.py | 40 +++++++++++++++++++-----
 2 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index a33f85970..4203c99ec 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -39,6 +39,7 @@ def __init__(self, data, net):
 
         # Backend-dependent attributes
         self.opt = None
+        self.lr_scheduler = None
         # Tensor or callable
         self.outputs = None
         self.outputs_losses_train = None
@@ -283,7 +284,7 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        self.opt = optimizers.get(
+        self.opt, self.lr_scheduler = optimizers.get(
             trainable_variables, self.opt_name, learning_rate=lr, decay=decay
         )
 
@@ -296,6 +297,8 @@ def closure():
                 return total_loss
 
             self.opt.step(closure)
+            if self.lr_scheduler is not None:
+                self.lr_scheduler.step()
 
         # Callables
         self.outputs = outputs
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index a96fa3339..5f9bae689 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -11,13 +11,15 @@ def is_external_optimizer(optimizer):
 
 def get(params, optimizer, learning_rate=None, decay=None):
     """Retrieves an Optimizer instance."""
+
+    #Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
-        return optimizer
+        optim =  optimizer
 
     if optimizer in ["L-BFGS", "L-BFGS-B"]:
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
-        return torch.optim.LBFGS(
+        optim =  torch.optim.LBFGS(
             params,
             lr=1,
             max_iter=LBFGS_options["iter_per_step"],
@@ -31,11 +33,33 @@ def get(params, optimizer, learning_rate=None, decay=None):
     if learning_rate is None:
         raise ValueError("No learning rate for {}.".format(optimizer))
 
+    # Default optimizer is Adam
+    optim = torch.optim.Adam(params, lr=learning_rate)
+
+    if optimizer == "SGD":
+        optim = torch.optim.SGD(params, lr = learning_rate)
+
+    if optimizer == "RMSprop":
+        optim = torch.optim.RMSprop(params,lr=learning_rate)
+
+    if optimizer not in ["SGD","RMSprop","adam"]:
+        raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
+
     if decay is not None:
-        # TODO: learning rate decay
+        lr_scheduler = _get_learningrate_scheduler(optim, decay)
+    else:
+        lr_scheduler = None
+
+    return optim, lr_scheduler
+
+def _get_learningrate_scheduler(optim, decay):
+    if decay is None:
+        return None
+
+    if decay[0] == "Step":
+        return torch.optim.lr_scheduler.StepLR(optim, step_size=decay[1], gamma=decay[2])
+    else:
+        # TODO: More learning rate scheduler
         raise NotImplementedError(
-            "learning rate decay to be implemented for backend pytorch."
-        )
-    if optimizer == "adam":
-        return torch.optim.Adam(params, lr=learning_rate)
-    raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
+            f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
+        )
\ No newline at end of file

From 6755256b89c412e344c2c445ba2909413d442fec Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 18:35:29 -0400
Subject: [PATCH 035/170] tiny change in pytorch/optimizers.py

---
 deepxde/optimizers/pytorch/optimizers.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 5f9bae689..8db6c9a90 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -58,8 +58,8 @@ def _get_learningrate_scheduler(optim, decay):
 
     if decay[0] == "Step":
         return torch.optim.lr_scheduler.StepLR(optim, step_size=decay[1], gamma=decay[2])
-    else:
-        # TODO: More learning rate scheduler
-        raise NotImplementedError(
-            f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
-        )
\ No newline at end of file
+
+    # TODO: More learning rate scheduler
+    raise NotImplementedError(
+        f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
+    )
\ No newline at end of file

From 5ded62c349985d3c1f9aa73f5743ab11cc25a2fb Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 18:44:45 -0400
Subject: [PATCH 036/170] Fix problems

---
 deepxde/model.py                         |  7 ++++--
 deepxde/optimizers/pytorch/optimizers.py | 27 ++++++++++++------------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 4203c99ec..cb0481d11 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -39,7 +39,6 @@ def __init__(self, data, net):
 
         # Backend-dependent attributes
         self.opt = None
-        self.lr_scheduler = None
         # Tensor or callable
         self.outputs = None
         self.outputs_losses_train = None
@@ -50,6 +49,8 @@ def __init__(self, data, net):
             self.saver = None
         elif backend_name == "jax":
             self.opt_state = None
+        elif backend_name == "pytorch":
+            self.lr_scheduler = None
 
     @utils.timing
     def compile(
@@ -252,7 +253,9 @@ def outputs(training, inputs):
         def outputs_losses(training, inputs, targets, losses_fn):
             self.net.train(mode=training)
             if isinstance(inputs, tuple):
-                inputs = tuple(map(lambda x: torch.as_tensor(x).requires_grad_(), inputs))
+                inputs = tuple(
+                    map(lambda x: torch.as_tensor(x).requires_grad_(), inputs)
+                )
             else:
                 inputs = torch.as_tensor(inputs)
                 inputs.requires_grad_()
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 8db6c9a90..2fd142e89 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -12,14 +12,14 @@ def is_external_optimizer(optimizer):
 def get(params, optimizer, learning_rate=None, decay=None):
     """Retrieves an Optimizer instance."""
 
-    #Custom Optimizer
+    # Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
-        optim =  optimizer
+        optim = optimizer
 
     if optimizer in ["L-BFGS", "L-BFGS-B"]:
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
-        optim =  torch.optim.LBFGS(
+        optim = torch.optim.LBFGS(
             params,
             lr=1,
             max_iter=LBFGS_options["iter_per_step"],
@@ -33,16 +33,14 @@ def get(params, optimizer, learning_rate=None, decay=None):
     if learning_rate is None:
         raise ValueError("No learning rate for {}.".format(optimizer))
 
-    # Default optimizer is Adam
-    optim = torch.optim.Adam(params, lr=learning_rate)
-
     if optimizer == "SGD":
-        optim = torch.optim.SGD(params, lr = learning_rate)
-
-    if optimizer == "RMSprop":
-        optim = torch.optim.RMSprop(params,lr=learning_rate)
+        optim = torch.optim.SGD(params, lr=learning_rate)
+    elif optimizer == "RMSprop":
+        optim = torch.optim.RMSprop(params, lr=learning_rate)
+    elif optimizer == "adam":
+        optim = torch.optim.Adam(params, lr=learning_rate)
 
-    if optimizer not in ["SGD","RMSprop","adam"]:
+    if optimizer not in ["SGD", "RMSprop", "adam"]:
         raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
 
     if decay is not None:
@@ -52,14 +50,17 @@ def get(params, optimizer, learning_rate=None, decay=None):
 
     return optim, lr_scheduler
 
+
 def _get_learningrate_scheduler(optim, decay):
     if decay is None:
         return None
 
     if decay[0] == "Step":
-        return torch.optim.lr_scheduler.StepLR(optim, step_size=decay[1], gamma=decay[2])
+        return torch.optim.lr_scheduler.StepLR(
+            optim, step_size=decay[1], gamma=decay[2]
+        )
 
     # TODO: More learning rate scheduler
     raise NotImplementedError(
         f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
-    )
\ No newline at end of file
+    )

From 7a3df950370cdb88bb6f843b9e931a74a1a5836a Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 19:09:16 -0400
Subject: [PATCH 037/170] tiny changes

---
 deepxde/model.py                         | 5 +++--
 deepxde/optimizers/pytorch/optimizers.py | 3 +--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index cb0481d11..593a69720 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -47,10 +47,11 @@ def __init__(self, data, net):
         if backend_name == "tensorflow.compat.v1":
             self.sess = None
             self.saver = None
-        elif backend_name == "jax":
-            self.opt_state = None
         elif backend_name == "pytorch":
             self.lr_scheduler = None
+        elif backend_name == "jax":
+            self.opt_state = None
+
 
     @utils.timing
     def compile(
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 2fd142e89..9ee741ba2 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -39,8 +39,7 @@ def get(params, optimizer, learning_rate=None, decay=None):
         optim = torch.optim.RMSprop(params, lr=learning_rate)
     elif optimizer == "adam":
         optim = torch.optim.Adam(params, lr=learning_rate)
-
-    if optimizer not in ["SGD", "RMSprop", "adam"]:
+    else:
         raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
 
     if decay is not None:

From e0031dfa04a01254a9d749731619f3557bd4ff09 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 19:58:46 -0400
Subject: [PATCH 038/170] Fix problem pytorch/optimizers.py

---
 deepxde/optimizers/pytorch/optimizers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 9ee741ba2..0bc398525 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -54,12 +54,12 @@ def _get_learningrate_scheduler(optim, decay):
     if decay is None:
         return None
 
-    if decay[0] == "Step":
+    if decay[0] == "step":
         return torch.optim.lr_scheduler.StepLR(
             optim, step_size=decay[1], gamma=decay[2]
         )
 
     # TODO: More learning rate scheduler
     raise NotImplementedError(
-        f"{decay[0]} learning rate scheduler to be implemented for backend tensorflow."
+        f"{decay[0]} learning rate scheduler to be implemented for backend pytorch."
     )

From 5c5893261aaa9ccaac0ddf39420be36e3b7ebeda Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 20:58:58 -0400
Subject: [PATCH 039/170] delete emply lines

---
 deepxde/model.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 593a69720..3ad39a28d 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -52,7 +52,6 @@ def __init__(self, data, net):
         elif backend_name == "jax":
             self.opt_state = None
 
-
     @utils.timing
     def compile(
         self,

From 04c01cc62416774a96e29cc5fc40068cb815299c Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 21:06:32 -0400
Subject: [PATCH 040/170] Changes in pytorch/optimizers.py

---
 deepxde/optimizers/pytorch/optimizers.py | 29 +++++++++++-------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 0bc398525..cf1b82587 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -11,12 +11,10 @@ def is_external_optimizer(optimizer):
 
 def get(params, optimizer, learning_rate=None, decay=None):
     """Retrieves an Optimizer instance."""
-
     # Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
         optim = optimizer
-
-    if optimizer in ["L-BFGS", "L-BFGS-B"]:
+    elif optimizer in ["L-BFGS", "L-BFGS-B"]:
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
         optim = torch.optim.LBFGS(
@@ -32,20 +30,19 @@ def get(params, optimizer, learning_rate=None, decay=None):
 
     if learning_rate is None:
         raise ValueError("No learning rate for {}.".format(optimizer))
-
-    if optimizer == "SGD":
-        optim = torch.optim.SGD(params, lr=learning_rate)
-    elif optimizer == "RMSprop":
-        optim = torch.optim.RMSprop(params, lr=learning_rate)
-    elif optimizer == "adam":
-        optim = torch.optim.Adam(params, lr=learning_rate)
-    else:
-        raise NotImplementedError(f"{optimizer} to be implemented for backend pytorch.")
-
-    if decay is not None:
-        lr_scheduler = _get_learningrate_scheduler(optim, decay)
     else:
-        lr_scheduler = None
+        if optimizer == "SGD":
+            optim = torch.optim.SGD(params, lr=learning_rate)
+        elif optimizer == "RMSprop":
+            optim = torch.optim.RMSprop(params, lr=learning_rate)
+        elif optimizer == "adam":
+            optim = torch.optim.Adam(params, lr=learning_rate)
+        else:
+            raise NotImplementedError(
+                f"{optimizer} to be implemented for backend pytorch."
+            )
+
+    lr_scheduler = _get_learningrate_scheduler(optim, decay)
 
     return optim, lr_scheduler
 

From 363bbd15e239545b943315d3f7f4905572d8a814 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 21:25:07 -0400
Subject: [PATCH 041/170] Fixed problem when optimizer in ["L-BFGS",
 "L-BFGS-B"] with learning_rate=None

---
 deepxde/optimizers/pytorch/optimizers.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index cf1b82587..fedecea41 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -27,12 +27,11 @@ def get(params, optimizer, learning_rate=None, decay=None):
             history_size=LBFGS_options["maxcor"],
             line_search_fn=None,
         )
-
-    if learning_rate is None:
-        raise ValueError("No learning rate for {}.".format(optimizer))
     else:
-        if optimizer == "SGD":
-            optim = torch.optim.SGD(params, lr=learning_rate)
+        if learning_rate is None:
+            raise ValueError("No learning rate for {}.".format(optimizer))
+        elif optimizer == "SGD":
+                optim = torch.optim.SGD(params, lr=learning_rate)
         elif optimizer == "RMSprop":
             optim = torch.optim.RMSprop(params, lr=learning_rate)
         elif optimizer == "adam":

From 9e7047999e57334eae4d75d4d96573e52337203f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 21:32:11 -0400
Subject: [PATCH 042/170] rename SGD as sgd, RMSprop as rmsprop

---
 deepxde/optimizers/pytorch/optimizers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index fedecea41..ebc0f8fcc 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -30,9 +30,9 @@ def get(params, optimizer, learning_rate=None, decay=None):
     else:
         if learning_rate is None:
             raise ValueError("No learning rate for {}.".format(optimizer))
-        elif optimizer == "SGD":
+        if optimizer == "sgd":
                 optim = torch.optim.SGD(params, lr=learning_rate)
-        elif optimizer == "RMSprop":
+        elif optimizer == "rmsprop":
             optim = torch.optim.RMSprop(params, lr=learning_rate)
         elif optimizer == "adam":
             optim = torch.optim.Adam(params, lr=learning_rate)

From f135384611c46b1adaa96f5ca337bc348f290f1d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 21:39:58 -0400
Subject: [PATCH 043/170] Fix intend problem

---
 deepxde/optimizers/pytorch/optimizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index ebc0f8fcc..171c89e2d 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -31,7 +31,7 @@ def get(params, optimizer, learning_rate=None, decay=None):
         if learning_rate is None:
             raise ValueError("No learning rate for {}.".format(optimizer))
         if optimizer == "sgd":
-                optim = torch.optim.SGD(params, lr=learning_rate)
+            optim = torch.optim.SGD(params, lr=learning_rate)
         elif optimizer == "rmsprop":
             optim = torch.optim.RMSprop(params, lr=learning_rate)
         elif optimizer == "adam":

From 627a610e551889daaee98ee75523cda17bf180d6 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 3 Jun 2022 21:43:13 -0400
Subject: [PATCH 044/170] delete empty lines

---
 deepxde/optimizers/pytorch/optimizers.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 171c89e2d..6a6dca441 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -40,9 +40,7 @@ def get(params, optimizer, learning_rate=None, decay=None):
             raise NotImplementedError(
                 f"{optimizer} to be implemented for backend pytorch."
             )
-
     lr_scheduler = _get_learningrate_scheduler(optim, decay)
-
     return optim, lr_scheduler
 
 

From f2eb4e9ddac51148becad6b8dc749956a9a20941 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 22 Jun 2022 21:29:25 -0400
Subject: [PATCH 045/170] add Pytorch implementation of AdamW optimizer

---
 deepxde/optimizers/pytorch/optimizers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 6a6dca441..cd4319cbd 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -36,6 +36,8 @@ def get(params, optimizer, learning_rate=None, decay=None):
             optim = torch.optim.RMSprop(params, lr=learning_rate)
         elif optimizer == "adam":
             optim = torch.optim.Adam(params, lr=learning_rate)
+        elif optimizer == "adamw":
+            optim = torch.optim.AdamW(params, lr=learning_rate)
         else:
             raise NotImplementedError(
                 f"{optimizer} to be implemented for backend pytorch."

From 0b3fbbef1bf120f391da2b026e75ff0ddb6fa83f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 22 Jun 2022 22:20:40 -0400
Subject: [PATCH 046/170] add Pytorch implementation of regularization

---
 deepxde/model.py                         |  8 +++++---
 deepxde/nn/regularizers.py               |  1 -
 deepxde/optimizers/pytorch/optimizers.py | 12 ++++++------
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index d1f79b99c..5456349b9 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -124,7 +124,11 @@ def compile(
         elif backend_name == "tensorflow":
             self._compile_tensorflow(lr, loss_fn, decay, loss_weights)
         elif backend_name == "pytorch":
-            self._compile_pytorch(lr, loss_fn, decay, loss_weights)
+            if decay == None:
+                decay = {"lr":None, "regular": 0}
+                self._compile_pytorch(lr, loss_fn, decay, loss_weights)
+            else:
+                self._compile_pytorch(lr, loss_fn,decay,loss_weights)
         elif backend_name == "jax":
             self._compile_jax(lr, loss_fn, decay, loss_weights)
         elif backend_name == "paddle":
@@ -252,7 +256,6 @@ def build_loss():
 
     def _compile_pytorch(self, lr, loss_fn, decay, loss_weights):
         """pytorch"""
-
         def outputs(training, inputs):
             self.net.train(mode=training)
             with torch.no_grad():
@@ -278,7 +281,6 @@ def outputs_losses(training, inputs, targets, losses_fn):
             losses = losses_fn(targets, outputs_, loss_fn, inputs, self)
             if not isinstance(losses, list):
                 losses = [losses]
-            # TODO: regularization
             losses = torch.stack(losses)
             # Weighted losses
             if loss_weights is not None:
diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py
index b3e4d868d..b87ac8d58 100644
--- a/deepxde/nn/regularizers.py
+++ b/deepxde/nn/regularizers.py
@@ -2,7 +2,6 @@
 
 
 def get(identifier):
-    # TODO: pytorch
     if identifier is None:
         return None
     name, scales = identifier[0], identifier[1:]
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index cd4319cbd..40a585f75 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -15,7 +15,7 @@ def get(params, optimizer, learning_rate=None, decay=None):
     if isinstance(optimizer, torch.optim.Optimizer):
         optim = optimizer
     elif optimizer in ["L-BFGS", "L-BFGS-B"]:
-        if learning_rate is not None or decay is not None:
+        if learning_rate is not None or decay["lr"] is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
         optim = torch.optim.LBFGS(
             params,
@@ -31,18 +31,18 @@ def get(params, optimizer, learning_rate=None, decay=None):
         if learning_rate is None:
             raise ValueError("No learning rate for {}.".format(optimizer))
         if optimizer == "sgd":
-            optim = torch.optim.SGD(params, lr=learning_rate)
+            optim = torch.optim.SGD(params, lr=learning_rate, weight_decay = decay["regular"])
         elif optimizer == "rmsprop":
-            optim = torch.optim.RMSprop(params, lr=learning_rate)
+            optim = torch.optim.RMSprop(params, lr=learning_rate, weight_decay = decay["regular"])
         elif optimizer == "adam":
-            optim = torch.optim.Adam(params, lr=learning_rate)
+            optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = decay["regular"])
         elif optimizer == "adamw":
-            optim = torch.optim.AdamW(params, lr=learning_rate)
+            optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = decay["regular"])
         else:
             raise NotImplementedError(
                 f"{optimizer} to be implemented for backend pytorch."
             )
-    lr_scheduler = _get_learningrate_scheduler(optim, decay)
+    lr_scheduler = _get_learningrate_scheduler(optim, decay["lr"])
     return optim, lr_scheduler
 
 

From 189f50c4f7893f819c9eea77c78c725ec4aac791 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 22 Jun 2022 23:39:20 -0400
Subject: [PATCH 047/170] add Pytorch implementation of l2 regularization

---
 deepxde/model.py                         | 25 +++++++++++------
 deepxde/nn/pytorch/deeponet.py           |  5 ++++
 deepxde/nn/pytorch/fnn.py                |  4 ++-
 deepxde/nn/pytorch/nn.py                 |  1 +
 deepxde/nn/regularizers.py               | 34 +++++++++++++++---------
 deepxde/nn/tensorflow/deeponet.py        | 10 ++++---
 deepxde/optimizers/pytorch/optimizers.py | 14 +++++-----
 7 files changed, 61 insertions(+), 32 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 5456349b9..7d94621b8 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -124,11 +124,7 @@ def compile(
         elif backend_name == "tensorflow":
             self._compile_tensorflow(lr, loss_fn, decay, loss_weights)
         elif backend_name == "pytorch":
-            if decay == None:
-                decay = {"lr":None, "regular": 0}
-                self._compile_pytorch(lr, loss_fn, decay, loss_weights)
-            else:
-                self._compile_pytorch(lr, loss_fn,decay,loss_weights)
+            self._compile_pytorch(lr, loss_fn, decay, loss_weights)
         elif backend_name == "jax":
             self._compile_jax(lr, loss_fn, decay, loss_weights)
         elif backend_name == "paddle":
@@ -276,6 +272,7 @@ def outputs_losses(training, inputs, targets, losses_fn):
                 inputs.requires_grad_()
             outputs_ = self.net(inputs)
             # Data losses
+            # TODO: l1 regularization and l1+l2 regularizaiton
             if targets is not None:
                 targets = torch.as_tensor(targets)
             losses = losses_fn(targets, outputs_, loss_fn, inputs, self)
@@ -301,9 +298,21 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        self.opt, self.lr_scheduler = optimizers.get(
-            trainable_variables, self.opt_name, learning_rate=lr, decay=decay
-        )
+        if self.net.regularization is None:
+            self.opt, self.lr_scheduler = optimizers.get(
+                trainable_variables, self.opt_name, learning_rate=lr, decay=decay
+            )
+        else:
+            if self.net.regularization[0] is 'l2':
+                self.opt, self.lr_scheduler = optimizers.get(
+                    trainable_variables, self.opt_name, learning_rate=lr,
+                    decay=decay, weight_decay=self.net.regularization[1]
+                )
+            else:
+                raise NotImplementedError(
+                    f"{self.net.regularization[0]} regularizaiton to be implemented for backend pytorch."
+                )
+
 
         def train_step(inputs, targets):
             def closure():
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index fb20d29c5..36410e78d 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -3,6 +3,7 @@
 from .fnn import FNN
 from .nn import NN
 from .. import activations
+from ..import regularizers
 
 
 class DeepONetCartesianProd(NN):
@@ -27,6 +28,7 @@ def __init__(
         layer_sizes_trunk,
         activation,
         kernel_initializer,
+        regularization=None
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -42,6 +44,7 @@ def __init__(
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
+        self.regularization = regularizers.get(regularization)
 
     def forward(self, inputs):
         x_func = inputs[0]
@@ -97,6 +100,7 @@ def __init__(
         activation,
         kernel_initializer,
         layer_sizes_trunk=None,
+        regularization = None
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -118,6 +122,7 @@ def __init__(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
             self.b = torch.tensor(0.0, requires_grad=True)
+        self.regularization = regularizers.get(regularization)
 
     def forward(self, inputs):
         x_func = inputs[0]
diff --git a/deepxde/nn/pytorch/fnn.py b/deepxde/nn/pytorch/fnn.py
index 31cb90378..d9a944590 100644
--- a/deepxde/nn/pytorch/fnn.py
+++ b/deepxde/nn/pytorch/fnn.py
@@ -3,14 +3,16 @@
 from .nn import NN
 from .. import activations
 from .. import initializers
+from .. import regularizers
 from ... import config
 
 
 class FNN(NN):
     """Fully-connected neural network."""
 
-    def __init__(self, layer_sizes, activation, kernel_initializer):
+    def __init__(self, layer_sizes, activation, kernel_initializer, regularization=None,):
         super().__init__()
+        self.regularizer = regularizers.get(regularization)
         self.activation = activations.get(activation)
         initializer = initializers.get(kernel_initializer)
         initializer_zero = initializers.get("zeros")
diff --git a/deepxde/nn/pytorch/nn.py b/deepxde/nn/pytorch/nn.py
index 01f1c30b5..51f56776d 100644
--- a/deepxde/nn/pytorch/nn.py
+++ b/deepxde/nn/pytorch/nn.py
@@ -6,6 +6,7 @@ class NN(torch.nn.Module):
 
     def __init__(self):
         super().__init__()
+        self.regularizer = None
         self._input_transform = None
         self._output_transform = None
 
diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py
index b87ac8d58..bb42f7b07 100644
--- a/deepxde/nn/regularizers.py
+++ b/deepxde/nn/regularizers.py
@@ -1,16 +1,26 @@
-from ..backend import tf
-
+from ..backend import backend_name, tf
 
 def get(identifier):
+    # TODO: other backends
     if identifier is None:
         return None
-    name, scales = identifier[0], identifier[1:]
-    return (
-        tf.keras.regularizers.l1(l=scales[0])
-        if name == "l1"
-        else tf.keras.regularizers.l2(l=scales[0])
-        if name == "l2"
-        else tf.keras.regularizers.l1_l2(l1=scales[0], l2=scales[1])
-        if name == "l1+l2"
-        else None
-    )
+    if backend_name == "pytorch":
+        return identifier
+    elif backend_name == "tensorflow" or "tensorflow.compat.v1":
+        name, scales = identifier[0], identifier[1:]
+        return (
+            tf.keras.regularizers.l1(l=scales[0])
+            if name == "l1"
+            else tf.keras.regularizers.l2(l=scales[0])
+            if name == "l2"
+            else tf.keras.regularizers.l1_l2(l1=scales[0], l2=scales[1])
+            if name == "l1+l2"
+            else None
+        )
+    else:
+        # TODO: other backends
+        raise NotImplementedError(
+            f"regularization to be implemented for backend {backend_name}."
+        )
+
+
diff --git a/deepxde/nn/tensorflow/deeponet.py b/deepxde/nn/tensorflow/deeponet.py
index 0d99f6704..50388f2c8 100644
--- a/deepxde/nn/tensorflow/deeponet.py
+++ b/deepxde/nn/tensorflow/deeponet.py
@@ -26,6 +26,7 @@ def __init__(
         layer_sizes_trunk,
         activation,
         kernel_initializer,
+        regularization=None
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -39,8 +40,8 @@ def __init__(
             self.branch = layer_sizes_branch[1]
         else:
             # Fully connected network
-            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
-        self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
+            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer, regularization)
+        self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization)
         self.b = tf.Variable(tf.zeros(1))
 
     def call(self, inputs, training=False):
@@ -98,6 +99,7 @@ def __init__(
         activation,
         kernel_initializer,
         layer_sizes_trunk=None,
+        regularization=None,
     ):
         super().__init__()
         self.pod_basis = tf.convert_to_tensor(pod_basis, dtype=tf.float32)
@@ -112,12 +114,12 @@ def __init__(
             self.branch = layer_sizes_branch[1]
         else:
             # Fully connected network
-            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
+            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer, regularization)
 
         self.trunk = None
         if layer_sizes_trunk is not None:
             self.trunk = FNN(
-                layer_sizes_trunk, self.activation_trunk, kernel_initializer
+                layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization
             )
             self.b = tf.Variable(tf.zeros(1))
 
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 40a585f75..c0e6394d3 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -9,13 +9,13 @@ def is_external_optimizer(optimizer):
     return optimizer in ["L-BFGS", "L-BFGS-B"]
 
 
-def get(params, optimizer, learning_rate=None, decay=None):
+def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
     """Retrieves an Optimizer instance."""
     # Custom Optimizer
     if isinstance(optimizer, torch.optim.Optimizer):
         optim = optimizer
     elif optimizer in ["L-BFGS", "L-BFGS-B"]:
-        if learning_rate is not None or decay["lr"] is not None:
+        if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
         optim = torch.optim.LBFGS(
             params,
@@ -31,18 +31,18 @@ def get(params, optimizer, learning_rate=None, decay=None):
         if learning_rate is None:
             raise ValueError("No learning rate for {}.".format(optimizer))
         if optimizer == "sgd":
-            optim = torch.optim.SGD(params, lr=learning_rate, weight_decay = decay["regular"])
+            optim = torch.optim.SGD(params, lr=learning_rate, weight_decay = weight_decay)
         elif optimizer == "rmsprop":
-            optim = torch.optim.RMSprop(params, lr=learning_rate, weight_decay = decay["regular"])
+            optim = torch.optim.RMSprop(params, lr=learning_rate, weight_decay = weight_decay)
         elif optimizer == "adam":
-            optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = decay["regular"])
+            optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = weight_decay)
         elif optimizer == "adamw":
-            optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = decay["regular"])
+            optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = weight_decay)
         else:
             raise NotImplementedError(
                 f"{optimizer} to be implemented for backend pytorch."
             )
-    lr_scheduler = _get_learningrate_scheduler(optim, decay["lr"])
+    lr_scheduler = _get_learningrate_scheduler(optim, decay)
     return optim, lr_scheduler
 
 

From cfd32547fd12e3bec5c9341b8912d58dd4e3c6f1 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 22 Jun 2022 23:44:20 -0400
Subject: [PATCH 048/170] tiny change of pytorch/fnn.py

---
 deepxde/nn/pytorch/fnn.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/deepxde/nn/pytorch/fnn.py b/deepxde/nn/pytorch/fnn.py
index d9a944590..31cb90378 100644
--- a/deepxde/nn/pytorch/fnn.py
+++ b/deepxde/nn/pytorch/fnn.py
@@ -3,16 +3,14 @@
 from .nn import NN
 from .. import activations
 from .. import initializers
-from .. import regularizers
 from ... import config
 
 
 class FNN(NN):
     """Fully-connected neural network."""
 
-    def __init__(self, layer_sizes, activation, kernel_initializer, regularization=None,):
+    def __init__(self, layer_sizes, activation, kernel_initializer):
         super().__init__()
-        self.regularizer = regularizers.get(regularization)
         self.activation = activations.get(activation)
         initializer = initializers.get(kernel_initializer)
         initializer_zero = initializers.get("zeros")

From 348a30e382f2b353e2f841fbdb2dc1cc78c5c9d1 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 00:06:55 -0400
Subject: [PATCH 049/170] tiny change of model.py and regularizers.py

---
 deepxde/model.py           |  2 +-
 deepxde/nn/regularizers.py | 27 +++++++++++----------------
 2 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 7d94621b8..f9cd48ba9 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -303,7 +303,7 @@ def outputs_losses_test(inputs, targets):
                 trainable_variables, self.opt_name, learning_rate=lr, decay=decay
             )
         else:
-            if self.net.regularization[0] is 'l2':
+            if self.net.regularization[0] == 'l2':
                 self.opt, self.lr_scheduler = optimizers.get(
                     trainable_variables, self.opt_name, learning_rate=lr,
                     decay=decay, weight_decay=self.net.regularization[1]
diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py
index bb42f7b07..b62659f86 100644
--- a/deepxde/nn/regularizers.py
+++ b/deepxde/nn/regularizers.py
@@ -6,21 +6,16 @@ def get(identifier):
         return None
     if backend_name == "pytorch":
         return identifier
-    elif backend_name == "tensorflow" or "tensorflow.compat.v1":
-        name, scales = identifier[0], identifier[1:]
-        return (
-            tf.keras.regularizers.l1(l=scales[0])
-            if name == "l1"
-            else tf.keras.regularizers.l2(l=scales[0])
-            if name == "l2"
-            else tf.keras.regularizers.l1_l2(l1=scales[0], l2=scales[1])
-            if name == "l1+l2"
-            else None
-        )
-    else:
-        # TODO: other backends
-        raise NotImplementedError(
-            f"regularization to be implemented for backend {backend_name}."
-        )
+    name, scales = identifier[0], identifier[1:]
+    return (
+        tf.keras.regularizers.l1(l=scales[0])
+        if name == "l1"
+        else tf.keras.regularizers.l2(l=scales[0])
+        if name == "l2"
+        else tf.keras.regularizers.l1_l2(l1=scales[0], l2=scales[1])
+        if name == "l1+l2"
+        else None
+    )
+
 
 

From 9105054e567959430be8bd80d75b3b3e18e1d28d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 00:16:08 -0400
Subject: [PATCH 050/170] delete Trailing newlines

---
 deepxde/nn/regularizers.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py
index b62659f86..587c796e8 100644
--- a/deepxde/nn/regularizers.py
+++ b/deepxde/nn/regularizers.py
@@ -16,6 +16,3 @@ def get(identifier):
         if name == "l1+l2"
         else None
     )
-
-
-

From 80f2632e0e37943e568f37ce4d73daabbd3b2d74 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 10:39:42 -0400
Subject: [PATCH 051/170] Fix problems

---
 deepxde/model.py                         | 10 ++++++----
 deepxde/nn/pytorch/deeponet.py           |  4 ++--
 deepxde/nn/regularizers.py               |  5 ++---
 deepxde/optimizers/pytorch/optimizers.py |  2 ++
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index f9cd48ba9..7cf17a144 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -303,17 +303,19 @@ def outputs_losses_test(inputs, targets):
                 trainable_variables, self.opt_name, learning_rate=lr, decay=decay
             )
         else:
-            if self.net.regularization[0] == 'l2':
+            if self.net.regularization[0] == "l2":
                 self.opt, self.lr_scheduler = optimizers.get(
-                    trainable_variables, self.opt_name, learning_rate=lr,
-                    decay=decay, weight_decay=self.net.regularization[1]
+                    trainable_variables,
+                    self.opt_name,
+                    learning_rate=lr,
+                    decay=decay,
+                    weight_decay=self.net.regularization[1],
                 )
             else:
                 raise NotImplementedError(
                     f"{self.net.regularization[0]} regularizaiton to be implemented for backend pytorch."
                 )
 
-
         def train_step(inputs, targets):
             def closure():
                 losses = outputs_losses_train(inputs, targets)[1]
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 36410e78d..16d8ff03f 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -44,7 +44,7 @@ def __init__(
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
-        self.regularization = regularizers.get(regularization)
+        self.regularization = regularization
 
     def forward(self, inputs):
         x_func = inputs[0]
@@ -122,7 +122,7 @@ def __init__(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
             self.b = torch.tensor(0.0, requires_grad=True)
-        self.regularization = regularizers.get(regularization)
+        self.regularization = regularization
 
     def forward(self, inputs):
         x_func = inputs[0]
diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py
index 587c796e8..a7e4de53d 100644
--- a/deepxde/nn/regularizers.py
+++ b/deepxde/nn/regularizers.py
@@ -1,11 +1,10 @@
-from ..backend import backend_name, tf
+from ..backend import tf
+
 
 def get(identifier):
     # TODO: other backends
     if identifier is None:
         return None
-    if backend_name == "pytorch":
-        return identifier
     name, scales = identifier[0], identifier[1:]
     return (
         tf.keras.regularizers.l1(l=scales[0])
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index c0e6394d3..0a3fec28b 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -37,6 +37,8 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
         elif optimizer == "adam":
             optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = weight_decay)
         elif optimizer == "adamw":
+            if weight_decay is 0:
+                raise ValueError("AdamW optimizer requires non-zero weight decay")
             optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = weight_decay)
         else:
             raise NotImplementedError(

From 2aba91cee71598e1c07bbb8633a01c0160c02b2e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 10:44:39 -0400
Subject: [PATCH 052/170] fix problem

---
 deepxde/nn/pytorch/deeponet.py           | 1 -
 deepxde/optimizers/pytorch/optimizers.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 16d8ff03f..973915608 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -3,7 +3,6 @@
 from .fnn import FNN
 from .nn import NN
 from .. import activations
-from ..import regularizers
 
 
 class DeepONetCartesianProd(NN):
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 0a3fec28b..5621fa801 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -37,7 +37,7 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
         elif optimizer == "adam":
             optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = weight_decay)
         elif optimizer == "adamw":
-            if weight_decay is 0:
+            if weight_decay == 0:
                 raise ValueError("AdamW optimizer requires non-zero weight decay")
             optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = weight_decay)
         else:

From 65ad905a95367397c1c4fa22e4bf3c18de155493 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 11:03:54 -0400
Subject: [PATCH 053/170] Reformat via black.

---
 deepxde/model.py                         |  1 +
 deepxde/nn/pytorch/deeponet.py           |  4 ++--
 deepxde/nn/tensorflow/deeponet.py        | 25 +++++++++++++++++++-----
 deepxde/optimizers/pytorch/optimizers.py | 14 +++++++++----
 4 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 7cf17a144..874bdc29a 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -252,6 +252,7 @@ def build_loss():
 
     def _compile_pytorch(self, lr, loss_fn, decay, loss_weights):
         """pytorch"""
+
         def outputs(training, inputs):
             self.net.train(mode=training)
             with torch.no_grad():
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 973915608..78322ada1 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -27,7 +27,7 @@ def __init__(
         layer_sizes_trunk,
         activation,
         kernel_initializer,
-        regularization=None
+        regularization=None,
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -99,7 +99,7 @@ def __init__(
         activation,
         kernel_initializer,
         layer_sizes_trunk=None,
-        regularization = None
+        regularization=None,
     ):
         super().__init__()
         if isinstance(activation, dict):
diff --git a/deepxde/nn/tensorflow/deeponet.py b/deepxde/nn/tensorflow/deeponet.py
index 50388f2c8..231253608 100644
--- a/deepxde/nn/tensorflow/deeponet.py
+++ b/deepxde/nn/tensorflow/deeponet.py
@@ -26,7 +26,7 @@ def __init__(
         layer_sizes_trunk,
         activation,
         kernel_initializer,
-        regularization=None
+        regularization=None,
     ):
         super().__init__()
         if isinstance(activation, dict):
@@ -40,8 +40,15 @@ def __init__(
             self.branch = layer_sizes_branch[1]
         else:
             # Fully connected network
-            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer, regularization)
-        self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization)
+            self.branch = FNN(
+                layer_sizes_branch,
+                activation_branch,
+                kernel_initializer,
+                regularization,
+            )
+        self.trunk = FNN(
+            layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization
+        )
         self.b = tf.Variable(tf.zeros(1))
 
     def call(self, inputs, training=False):
@@ -114,12 +121,20 @@ def __init__(
             self.branch = layer_sizes_branch[1]
         else:
             # Fully connected network
-            self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer, regularization)
+            self.branch = FNN(
+                layer_sizes_branch,
+                activation_branch,
+                kernel_initializer,
+                regularization,
+            )
 
         self.trunk = None
         if layer_sizes_trunk is not None:
             self.trunk = FNN(
-                layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization
+                layer_sizes_trunk,
+                self.activation_trunk,
+                kernel_initializer,
+                regularization,
             )
             self.b = tf.Variable(tf.zeros(1))
 
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 5621fa801..9d0e366c7 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -31,15 +31,21 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
         if learning_rate is None:
             raise ValueError("No learning rate for {}.".format(optimizer))
         if optimizer == "sgd":
-            optim = torch.optim.SGD(params, lr=learning_rate, weight_decay = weight_decay)
+            optim = torch.optim.SGD(params, lr=learning_rate, weight_decay=weight_decay)
         elif optimizer == "rmsprop":
-            optim = torch.optim.RMSprop(params, lr=learning_rate, weight_decay = weight_decay)
+            optim = torch.optim.RMSprop(
+                params, lr=learning_rate, weight_decay=weight_decay
+            )
         elif optimizer == "adam":
-            optim = torch.optim.Adam(params, lr=learning_rate, weight_decay = weight_decay)
+            optim = torch.optim.Adam(
+                params, lr=learning_rate, weight_decay=weight_decay
+            )
         elif optimizer == "adamw":
             if weight_decay == 0:
                 raise ValueError("AdamW optimizer requires non-zero weight decay")
-            optim = torch.optim.AdamW(params, lr=learning_rate, weight_decay = weight_decay)
+            optim = torch.optim.AdamW(
+                params, lr=learning_rate, weight_decay=weight_decay
+            )
         else:
             raise NotImplementedError(
                 f"{optimizer} to be implemented for backend pytorch."

From 4d79150ab2e2cb22a60de777d86b1a621f58c745 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 11:06:52 -0400
Subject: [PATCH 054/170] delete TODO

---
 deepxde/model.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 874bdc29a..2f5e69c4e 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -273,7 +273,6 @@ def outputs_losses(training, inputs, targets, losses_fn):
                 inputs.requires_grad_()
             outputs_ = self.net(inputs)
             # Data losses
-            # TODO: l1 regularization and l1+l2 regularizaiton
             if targets is not None:
                 targets = torch.as_tensor(targets)
             losses = losses_fn(targets, outputs_, loss_fn, inputs, self)

From 10ff987d2b580b39dc7c655c717302caca3918c3 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 11:19:50 -0400
Subject: [PATCH 055/170] self.regularizer = regularization

---
 deepxde/model.py               | 8 ++++----
 deepxde/nn/pytorch/deeponet.py | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 2f5e69c4e..22f7b5d40 100644
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -298,22 +298,22 @@ def outputs_losses_test(inputs, targets):
         trainable_variables = (
             list(self.net.parameters()) + self.external_trainable_variables
         )
-        if self.net.regularization is None:
+        if self.net.regularizer is None:
             self.opt, self.lr_scheduler = optimizers.get(
                 trainable_variables, self.opt_name, learning_rate=lr, decay=decay
             )
         else:
-            if self.net.regularization[0] == "l2":
+            if self.net.regularizer[0] == "l2":
                 self.opt, self.lr_scheduler = optimizers.get(
                     trainable_variables,
                     self.opt_name,
                     learning_rate=lr,
                     decay=decay,
-                    weight_decay=self.net.regularization[1],
+                    weight_decay=self.net.regularizer[1],
                 )
             else:
                 raise NotImplementedError(
-                    f"{self.net.regularization[0]} regularizaiton to be implemented for backend pytorch."
+                    f"{self.net.regularizer[0]} regularizaiton to be implemented for backend pytorch."
                 )
 
         def train_step(inputs, targets):
diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 78322ada1..178b26012 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -43,7 +43,7 @@ def __init__(
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
-        self.regularization = regularization
+        self.regularizer = regularization
 
     def forward(self, inputs):
         x_func = inputs[0]
@@ -121,7 +121,7 @@ def __init__(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
             self.b = torch.tensor(0.0, requires_grad=True)
-        self.regularization = regularization
+        self.regularizer = regularization
 
     def forward(self, inputs):
         x_func = inputs[0]

From 1986884c8ff66437c7fe308d8f8850ccd018e243 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 11:31:43 -0400
Subject: [PATCH 056/170] Modify regularization, to
 regularization=regularization, and For L-BFGS, raise an error if weigth_decay
 > 0

---
 deepxde/nn/tensorflow/deeponet.py        | 11 +++++++----
 deepxde/optimizers/pytorch/optimizers.py |  2 ++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/deepxde/nn/tensorflow/deeponet.py b/deepxde/nn/tensorflow/deeponet.py
index 231253608..117bfff04 100644
--- a/deepxde/nn/tensorflow/deeponet.py
+++ b/deepxde/nn/tensorflow/deeponet.py
@@ -44,10 +44,13 @@ def __init__(
                 layer_sizes_branch,
                 activation_branch,
                 kernel_initializer,
-                regularization,
+                regularization=regularization,
             )
         self.trunk = FNN(
-            layer_sizes_trunk, self.activation_trunk, kernel_initializer, regularization
+            layer_sizes_trunk,
+            self.activation_trunk,
+            kernel_initializer,
+            regularization=regularization,
         )
         self.b = tf.Variable(tf.zeros(1))
 
@@ -125,7 +128,7 @@ def __init__(
                 layer_sizes_branch,
                 activation_branch,
                 kernel_initializer,
-                regularization,
+                regularization=regularization,
             )
 
         self.trunk = None
@@ -134,7 +137,7 @@ def __init__(
                 layer_sizes_trunk,
                 self.activation_trunk,
                 kernel_initializer,
-                regularization,
+                regularization=regularization,
             )
             self.b = tf.Variable(tf.zeros(1))
 
diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 9d0e366c7..188b6725d 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -15,6 +15,8 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
     if isinstance(optimizer, torch.optim.Optimizer):
         optim = optimizer
     elif optimizer in ["L-BFGS", "L-BFGS-B"]:
+        if weight_decay > 0:
+            raise ValueError("L-BFGS optimizer doesn't require weight_decay > 0")
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
         optim = torch.optim.LBFGS(

From 327d51c98cdbe0b73349589f808f81640ead7c32 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 23 Jun 2022 11:38:14 -0400
Subject: [PATCH 057/170] doesn't require --> doesn't support

---
 deepxde/optimizers/pytorch/optimizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py
index 188b6725d..21b0d3fff 100644
--- a/deepxde/optimizers/pytorch/optimizers.py
+++ b/deepxde/optimizers/pytorch/optimizers.py
@@ -16,7 +16,7 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0):
         optim = optimizer
     elif optimizer in ["L-BFGS", "L-BFGS-B"]:
         if weight_decay > 0:
-            raise ValueError("L-BFGS optimizer doesn't require weight_decay > 0")
+            raise ValueError("L-BFGS optimizer doesn't support weight_decay > 0")
         if learning_rate is not None or decay is not None:
             print("Warning: learning rate is ignored for {}".format(optimizer))
         optim = torch.optim.LBFGS(

From 7c012788cc1db7b4879d15c261d6b2db991d70e9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 15:12:38 -0400
Subject: [PATCH 058/170] backend Pytorch support MIONetCartesianProd and
 PODMIONet

---
 deepxde/nn/pytorch/__init__.py |   3 +-
 deepxde/nn/pytorch/mionet.py   | 154 +++++++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+), 1 deletion(-)
 create mode 100644 deepxde/nn/pytorch/mionet.py

diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index bebdc53d6..8d9ce86ef 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -1,7 +1,8 @@
 """Package for pytorch NN modules."""
 
-__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet"]
+__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet","MIONetCartesianProd"]
 
 from .deeponet import DeepONetCartesianProd, PODDeepONet
+from .mionet import MIONetCartesianProd
 from .fnn import FNN, PFNN
 from .nn import NN
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
new file mode 100644
index 000000000..704163796
--- /dev/null
+++ b/deepxde/nn/pytorch/mionet.py
@@ -0,0 +1,154 @@
+import torch
+from .nn import NN
+from .fnn import FNN
+from .. import activations
+from .. import initializers
+from .. import regularizers
+from ... import config
+from ...utils import timing
+
+
+class MIONetCartesianProd(NN):
+    """MIONet with two input functions and proper orthogonal decomposition (POD) for Cartesian product format."""
+
+    def __init__(
+        self,
+        layer_sizes_branch1,
+        layer_sizes_branch2,
+        layer_sizes_trunk,
+        activation,
+        kernel_initializer,
+        regularization=None,
+    ):
+        super().__init__()
+
+        if isinstance(activation, dict):
+            self.activation_branch1 = activations.get(activation["branch1"])
+            self.activation_branch2 = activations.get(activation["branch2"])
+            self.activation_trunk = activations.get(activation["trunk"])
+        else:
+            self.activation_branch1 = (
+                self.activation_branch2
+            ) = self.activation_trunk = activations.get(activation)
+        if callable(layer_sizes_branch1[1]):
+            # User-defined network
+            self.branch1 = layer_sizes_branch1[1]
+        else:
+            # Fully connected network
+            self.branch1 = FNN(
+                layer_sizes_branch1, self.activation_branch1, kernel_initializer
+            )
+        if callable(layer_sizes_branch2[1]):
+            # User-defined network
+            self.branch2 = layer_sizes_branch2[1]
+        else:
+            # Fully connected network
+            self.branch2 = FNN(
+                layer_sizes_branch2, self.activation_branch2, kernel_initializer
+            )
+        self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.b = torch.tensor(0.0, requires_grad=True)
+        self.regularizer = regularization
+
+    def forward(self, input):
+        x_func1 = input[0]
+        x_func2 = input[1]
+        x_loc = input[2]
+        # Branch net to encode the input function
+        y_func1 = self.branch1(x_func1)
+        y_func2 = self.branch2(x_func2)
+        # Trunk net to encode the domain of the output function
+        if self._input_transform is not None:
+            y_loc = self._input_transform(x_loc)
+        y_loc = self.activation_trunk(self.trunk(x_loc))
+        # Dot product
+        # Dot product
+        if y_func1.shape[-1] != y_func2.shape[-1]:
+            raise AssertionError(
+                "Output sizes of branch1 net and branch2 net do not match."
+            )
+        y = torch.mul(y_func1, y_func2)
+        if y.shape[-1] != y_loc.shape[-1]:
+            raise AssertionError(
+                "Output sizes of branch net and trunk net do not match."
+            )
+        y = torch.einsum("ip,jp->ij", y, y_loc)
+        # Add bias
+        y += self.b
+        if self._output_transform is not None:
+            y = self._output_transform(input, y)
+        return y
+
+
+class PODMIONet(NN):
+    """MIONet with two input functions for Cartesian product format."""
+
+    def __init__(
+        self,
+        pod_basis,
+        layer_sizes_branch1,
+        layer_sizes_branch2,
+        activation,
+        kernel_initializer,
+        layer_sizes_trunk=None,
+        regularization=None,
+    ):
+        super().__init__()
+
+        if isinstance(activation, dict):
+            self.activation_branch1 = activations.get(activation["branch1"])
+            self.activation_branch2 = activations.get(activation["branch2"])
+            self.activation_trunk = activations.get(activation["trunk"])
+        else:
+            self.activation_branch1 = (
+                self.activation_branch2
+            ) = self.activation_trunk = activations.get(activation)
+        self.pod_basis = torch.as_tensor(pod_basis, dtype=torch.float32)
+        if callable(layer_sizes_branch1[1]):
+            # User-defined network
+            self.branch1 = layer_sizes_branch1[1]
+        else:
+            # Fully connected network
+            self.branch1 = FNN(
+                layer_sizes_branch1, self.activation_branch1, kernel_initializer
+            )
+        if callable(layer_sizes_branch2[1]):
+            # User-defined network
+            self.branch2 = layer_sizes_branch2[1]
+        else:
+            # Fully connected network
+            self.branch2 = FNN(
+                layer_sizes_branch2, self.activation_branch2, kernel_initializer
+            )
+        self.trunk = None
+        if layer_sizes_trunk is not None:
+            self.trunk = FNN(
+                layer_sizes_trunk, self.activation_trunk, kernel_initializer
+            )
+            self.b = torch.tensor(0.0, requires_grad=True)
+        self.regularizer = regularization
+
+    def forward(self, input):
+        x_func1 = input[0]
+        x_func2 = input[1]
+        x_loc = input[2]
+        # Branch net to encode the input function
+        y_func1 = self.branch1(x_func1)
+        y_func2 = self.branch2(x_func2)
+        # Dot product
+        if y_func1.shape[-1] != y_func2.shape[-1]:
+            raise AssertionError(
+                "Output sizes of branch1 net and branch2 net do not match."
+            )
+        y_func = torch.mul(y_func1, y_func2)
+        if self.trunk is None:
+            # POD only
+            y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
+        else:
+            y_loc = self.activation_trunk(self.trunk(x_loc))
+            y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
+            y += self.b
+        if self._output_transform is not None:
+            y = self._output_transform(input, y)
+        return y

From 907557acb80dcf1bc4a69ac83b6645f1d42f6291 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 15:21:33 -0400
Subject: [PATCH 059/170] delete repeated comments"Dot product"

---
 deepxde/nn/pytorch/mionet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 704163796..1d47efcef 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -63,7 +63,6 @@ def forward(self, input):
             y_loc = self._input_transform(x_loc)
         y_loc = self.activation_trunk(self.trunk(x_loc))
         # Dot product
-        # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(
                 "Output sizes of branch1 net and branch2 net do not match."

From e1da975be1c7d214aaba7dade3cc74e01b787840 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 15:24:27 -0400
Subject: [PATCH 060/170] change nn/pytorch/__init__.py

---
 deepxde/nn/pytorch/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index 8d9ce86ef..90f8c81ab 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -1,8 +1,8 @@
 """Package for pytorch NN modules."""
 
-__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet","MIONetCartesianProd"]
+__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet","PODMIONet","MIONetCartesianProd"]
 
 from .deeponet import DeepONetCartesianProd, PODDeepONet
-from .mionet import MIONetCartesianProd
+from .mionet import MIONetCartesianProd, PODMIONet
 from .fnn import FNN, PFNN
 from .nn import NN

From 4c020fba4c852f865fab3a84a9f9de2893f0f171 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 15:34:38 -0400
Subject: [PATCH 061/170] reformat via black

---
 deepxde/nn/pytorch/__init__.py | 10 +++++++++-
 deepxde/nn/pytorch/mionet.py   |  6 +-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index 90f8c81ab..ff0c5f132 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -1,6 +1,14 @@
 """Package for pytorch NN modules."""
 
-__all__ = ["DeepONetCartesianProd", "FNN", "NN", "PFNN", "PODDeepONet","PODMIONet","MIONetCartesianProd"]
+__all__ = [
+    "DeepONetCartesianProd",
+    "FNN",
+    "NN",
+    "PFNN",
+    "PODDeepONet",
+    "PODMIONet",
+    "MIONetCartesianProd",
+]
 
 from .deeponet import DeepONetCartesianProd, PODDeepONet
 from .mionet import MIONetCartesianProd, PODMIONet
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 1d47efcef..823e62ce3 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -1,11 +1,8 @@
 import torch
+
 from .nn import NN
 from .fnn import FNN
 from .. import activations
-from .. import initializers
-from .. import regularizers
-from ... import config
-from ...utils import timing
 
 
 class MIONetCartesianProd(NN):
@@ -47,7 +44,6 @@ def __init__(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.kernel_initializer = initializers.get(kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
 

From 9e222c38c0983c8adde656eb57e8137c4478c5dc Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 15:40:33 -0400
Subject: [PATCH 062/170] change input to inputs

---
 deepxde/nn/pytorch/mionet.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 823e62ce3..3e71a8499 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -47,10 +47,10 @@ def __init__(
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
 
-    def forward(self, input):
-        x_func1 = input[0]
-        x_func2 = input[1]
-        x_loc = input[2]
+    def forward(self, inputs):
+        x_func1 = inputs[0]
+        x_func2 = inputs[1]
+        x_loc = inputs[2]
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
@@ -72,7 +72,7 @@ def forward(self, input):
         # Add bias
         y += self.b
         if self._output_transform is not None:
-            y = self._output_transform(input, y)
+            y = self._output_transform(inputs, y)
         return y
 
 
@@ -124,10 +124,10 @@ def __init__(
             self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
 
-    def forward(self, input):
-        x_func1 = input[0]
-        x_func2 = input[1]
-        x_loc = input[2]
+    def forward(self, inputs):
+        x_func1 = inputs[0]
+        x_func2 = inputs[1]
+        x_loc = inputs[2]
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
@@ -145,5 +145,5 @@ def forward(self, input):
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b
         if self._output_transform is not None:
-            y = self._output_transform(input, y)
+            y = self._output_transform(inputs, y)
         return y

From 47f6eaf802509a7f0a30937fae5a5b5b981a3e1e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 16:10:58 -0400
Subject: [PATCH 063/170] fix doc typo

---
 PODMIONet_training.py          | 91 ++++++++++++++++++++++++++++++++++
 deepxde/nn/pytorch/__init__.py |  2 +-
 deepxde/nn/pytorch/mionet.py   |  5 +-
 3 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100644 PODMIONet_training.py

diff --git a/PODMIONet_training.py b/PODMIONet_training.py
new file mode 100644
index 000000000..88dc146c2
--- /dev/null
+++ b/PODMIONet_training.py
@@ -0,0 +1,91 @@
+import deepxde as dde
+from deepxde.nn.pytorch.mionet import MIONetCartesianProd,PODMIONet
+# from deepxde.nn.tensorflow_compat_v1.deeponet import DeepONet, DeepONetCartesianProd
+from deepxde.data.quadruple import Quadruple, QuadrupleCartesianProd
+from deepxde.data.triple import Triple, TripleCartesianProd
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import tensorflow.compat.v1 as tf
+def pod(y):
+    n = len(y)
+    y_mean = np.mean(y, axis=0)
+    y = y - y_mean
+    C = 1 / (n - 1) * y.T @ y
+    w, v = np.linalg.eigh(C)
+    w = np.flip(w)
+    v = np.fliplr(v)
+    v *= len(y_mean) ** 0.5
+    # w_cumsum = np.cumsum(w)
+    # print(w_cumsum[:16] / w_cumsum[-1])
+    # plt.figure()
+    # plt.plot(y_mean)
+    # plt.figure()
+    # for i in range(8):
+    #     plt.subplot(2, 4, i + 1)
+    #     plt.plot(v[:, i])
+    # plt.show()
+    v = np.ascontiguousarray(v)
+    return y_mean, v
+
+def network(problem, m):
+    if problem == "ODE":
+        branch = [m, 200, 200]
+        trunk = [1, 200, 200]
+    elif problem == "DR":
+        branch = [m, 200, 200]
+        trunk = [2, 200, 200]
+    elif problem == "ADVD":
+        branch = [m, 300, 300, 32]
+        trunk = [2, 300, 300, 32]
+    return branch, trunk
+
+
+def run(problem, lr, epochs, m, activation, initializer):
+    training_data = np.load(problem + '_train.npz', allow_pickle=True)
+    testing_data = np.load(problem + '_test.npz', allow_pickle=True)
+    X_train = training_data['X_train']
+    y_train = training_data['y_train'].astype(np.float32)
+    X_test = testing_data['X_test']
+    y_test = testing_data['y_test'].astype(np.float32)
+    X_train = (X_train[0].astype(np.float32), X_train[1].astype(np.float32), np.array(X_train[2]).astype(np.float32))
+    X_test = (X_test[0].astype(np.float32), X_test[1].astype(np.float32), np.array(X_test[2]).astype(np.float32))
+
+    branch_net, trunk_net = network(problem, m)
+    y_mean, v = pod(y_test)
+    data = QuadrupleCartesianProd(X_train, y_train, X_test, y_test)
+    # net = MIONetCartesianProd(branch_net, branch_net, trunk_net,
+    #                           {"branch1": activation[0],
+    #                            "branch2": activation[1],
+    #                            "trunk": activation[2]},
+    #                           initializer, regularization=None)
+    net = PODMIONet(v[:, :32],branch_net,branch_net,
+                    {"branch1":activation[0],
+                    "branch2":activation[1],
+                    "trunk":activation[2]},
+                    initializer, regularization=None)
+    model = dde.Model(data, net)
+    model.compile("adam", lr=lr)
+    checker = dde.callbacks.ModelCheckpoint("model/mionet_model.ckpt", save_better_only=True, period=1000)
+    losshistory, train_state = model.train(epochs=epochs, callbacks=[checker])
+    # print("# Parameters:", np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]))
+
+
+def main():
+    # Problems:
+    # - "ODE": Antiderivative, Nonlinear ODE
+    # - "DR": Diffusion-reaction
+    # - "ADVD": Advection-diffusion
+    problem = "ADVD"
+    T = 1
+    m = 100
+    lr = 0.0002 if problem in ["ADVD"] else 0.001
+    epochs = 100000
+    activation = ["relu", None, "relu"] if problem in ["ADVD"] else ["relu", "relu", "relu"]
+    initializer = "Glorot normal"
+
+    run(problem, lr, epochs, m, activation, initializer)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/deepxde/nn/pytorch/__init__.py b/deepxde/nn/pytorch/__init__.py
index ff0c5f132..480111b10 100755
--- a/deepxde/nn/pytorch/__init__.py
+++ b/deepxde/nn/pytorch/__init__.py
@@ -3,11 +3,11 @@
 __all__ = [
     "DeepONetCartesianProd",
     "FNN",
+    "MIONetCartesianProd",
     "NN",
     "PFNN",
     "PODDeepONet",
     "PODMIONet",
-    "MIONetCartesianProd",
 ]
 
 from .deeponet import DeepONetCartesianProd, PODDeepONet
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 3e71a8499..5a5f974c8 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -6,7 +6,7 @@
 
 
 class MIONetCartesianProd(NN):
-    """MIONet with two input functions and proper orthogonal decomposition (POD) for Cartesian product format."""
+    """MIONet with two input functions for Cartesian product format."""
 
     def __init__(
         self,
@@ -77,7 +77,8 @@ def forward(self, inputs):
 
 
 class PODMIONet(NN):
-    """MIONet with two input functions for Cartesian product format."""
+    """MIONet with two input functions and proper orthogonal decomposition (POD)
+    for Cartesian product format."""
 
     def __init__(
         self,

From 7ddf1ad801b975b868e72f41eee5fafb7c271aeb Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 16:18:06 -0400
Subject: [PATCH 064/170] git rm PODMIO

---
 PODMIONet_training.py | 91 -------------------------------------------
 1 file changed, 91 deletions(-)
 delete mode 100644 PODMIONet_training.py

diff --git a/PODMIONet_training.py b/PODMIONet_training.py
deleted file mode 100644
index 88dc146c2..000000000
--- a/PODMIONet_training.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import deepxde as dde
-from deepxde.nn.pytorch.mionet import MIONetCartesianProd,PODMIONet
-# from deepxde.nn.tensorflow_compat_v1.deeponet import DeepONet, DeepONetCartesianProd
-from deepxde.data.quadruple import Quadruple, QuadrupleCartesianProd
-from deepxde.data.triple import Triple, TripleCartesianProd
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-import tensorflow.compat.v1 as tf
-def pod(y):
-    n = len(y)
-    y_mean = np.mean(y, axis=0)
-    y = y - y_mean
-    C = 1 / (n - 1) * y.T @ y
-    w, v = np.linalg.eigh(C)
-    w = np.flip(w)
-    v = np.fliplr(v)
-    v *= len(y_mean) ** 0.5
-    # w_cumsum = np.cumsum(w)
-    # print(w_cumsum[:16] / w_cumsum[-1])
-    # plt.figure()
-    # plt.plot(y_mean)
-    # plt.figure()
-    # for i in range(8):
-    #     plt.subplot(2, 4, i + 1)
-    #     plt.plot(v[:, i])
-    # plt.show()
-    v = np.ascontiguousarray(v)
-    return y_mean, v
-
-def network(problem, m):
-    if problem == "ODE":
-        branch = [m, 200, 200]
-        trunk = [1, 200, 200]
-    elif problem == "DR":
-        branch = [m, 200, 200]
-        trunk = [2, 200, 200]
-    elif problem == "ADVD":
-        branch = [m, 300, 300, 32]
-        trunk = [2, 300, 300, 32]
-    return branch, trunk
-
-
-def run(problem, lr, epochs, m, activation, initializer):
-    training_data = np.load(problem + '_train.npz', allow_pickle=True)
-    testing_data = np.load(problem + '_test.npz', allow_pickle=True)
-    X_train = training_data['X_train']
-    y_train = training_data['y_train'].astype(np.float32)
-    X_test = testing_data['X_test']
-    y_test = testing_data['y_test'].astype(np.float32)
-    X_train = (X_train[0].astype(np.float32), X_train[1].astype(np.float32), np.array(X_train[2]).astype(np.float32))
-    X_test = (X_test[0].astype(np.float32), X_test[1].astype(np.float32), np.array(X_test[2]).astype(np.float32))
-
-    branch_net, trunk_net = network(problem, m)
-    y_mean, v = pod(y_test)
-    data = QuadrupleCartesianProd(X_train, y_train, X_test, y_test)
-    # net = MIONetCartesianProd(branch_net, branch_net, trunk_net,
-    #                           {"branch1": activation[0],
-    #                            "branch2": activation[1],
-    #                            "trunk": activation[2]},
-    #                           initializer, regularization=None)
-    net = PODMIONet(v[:, :32],branch_net,branch_net,
-                    {"branch1":activation[0],
-                    "branch2":activation[1],
-                    "trunk":activation[2]},
-                    initializer, regularization=None)
-    model = dde.Model(data, net)
-    model.compile("adam", lr=lr)
-    checker = dde.callbacks.ModelCheckpoint("model/mionet_model.ckpt", save_better_only=True, period=1000)
-    losshistory, train_state = model.train(epochs=epochs, callbacks=[checker])
-    # print("# Parameters:", np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]))
-
-
-def main():
-    # Problems:
-    # - "ODE": Antiderivative, Nonlinear ODE
-    # - "DR": Diffusion-reaction
-    # - "ADVD": Advection-diffusion
-    problem = "ADVD"
-    T = 1
-    m = 100
-    lr = 0.0002 if problem in ["ADVD"] else 0.001
-    epochs = 100000
-    activation = ["relu", None, "relu"] if problem in ["ADVD"] else ["relu", "relu", "relu"]
-    initializer = "Glorot normal"
-
-    run(problem, lr, epochs, m, activation, initializer)
-
-
-if __name__ == "__main__":
-    main()

From 6cb1bdd5cfafec94e164dd4055cb7b3ca16d95e6 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 17:54:09 -0400
Subject: [PATCH 065/170] little change in mionet.py

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 5a5f974c8..33e54112b 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -1,7 +1,7 @@
 import torch
 
-from .nn import NN
 from .fnn import FNN
+from .nn import NN
 from .. import activations
 
 

From b5c0b2a5ed780b98ace87111a14a083833f3f4bc Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 18:09:06 -0400
Subject: [PATCH 066/170] branch1 net --> branch net1 branch2 net --> branch
 net2

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 33e54112b..939e04f0d 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -61,7 +61,7 @@ def forward(self, inputs):
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(
-                "Output sizes of branch1 net and branch2 net do not match."
+                "Output sizes of branch net1 and branch net2 do not match."
             )
         y = torch.mul(y_func1, y_func2)
         if y.shape[-1] != y_loc.shape[-1]:

From 3543793a97ecbecf731e3973343759cb6a6986de Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 18:14:13 -0400
Subject: [PATCH 067/170] y_loc = self.trunk(x_loc)

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 939e04f0d..be1897245 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -57,7 +57,7 @@ def forward(self, inputs):
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
-        y_loc = self.activation_trunk(self.trunk(x_loc))
+        y_loc = self.trunk(x_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(

From 2c524797093c660fc8b8216291f0f383846ba58a Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 18:16:00 -0400
Subject: [PATCH 068/170] y_loc = self.trunk(x_loc)

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index be1897245..45ec8c26e 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -142,7 +142,7 @@ def forward(self, inputs):
             # POD only
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
-            y_loc = self.activation_trunk(self.trunk(x_loc))
+            y_loc = self.trunk(x_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b
         if self._output_transform is not None:

From 96e28d65ba569899b7ea8cc534fde5745d06c05b Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 21:07:01 -0400
Subject: [PATCH 069/170] add trunk_activation:bool to determine whether use
 activation func in trunk net

---
 deepxde/nn/pytorch/mionet.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 45ec8c26e..8056bec67 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -16,6 +16,7 @@ def __init__(
         activation,
         kernel_initializer,
         regularization=None,
+        trunk_activation=False,
     ):
         super().__init__()
 
@@ -46,6 +47,7 @@ def __init__(
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
+        self.trunk_activation = trunk_activation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -57,7 +59,10 @@ def forward(self, inputs):
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
-        y_loc = self.trunk(x_loc)
+        if self.trunk_activation:
+            y_loc = self.activation_trunk(self.trunk(x_loc))
+        else:
+            y_loc = self.trunk(x_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(
@@ -89,6 +94,7 @@ def __init__(
         kernel_initializer,
         layer_sizes_trunk=None,
         regularization=None,
+        trunk_activaiton=False,
     ):
         super().__init__()
 
@@ -124,6 +130,7 @@ def __init__(
             )
             self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
+        self.trunk_activation = trunk_activaiton
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -142,7 +149,10 @@ def forward(self, inputs):
             # POD only
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
-            y_loc = self.trunk(x_loc)
+            if self.trunk_activation:
+                y_loc = self.activation_trunk(self.trunk(x_loc))
+            else:
+                y_loc = self.trunk(x_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b
         if self._output_transform is not None:

From cadeed9dc5d5f2e677bf4ab0b1cd371bb020b607 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 21:14:01 -0400
Subject: [PATCH 070/170] add trunk_last_activation:bool to determine whether
 use activation func in trunk net

---
 deepxde/nn/pytorch/mionet.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 8056bec67..da5858cc6 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -16,7 +16,7 @@ def __init__(
         activation,
         kernel_initializer,
         regularization=None,
-        trunk_activation=False,
+        trunk_last_activation=False,
     ):
         super().__init__()
 
@@ -47,7 +47,7 @@ def __init__(
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
-        self.trunk_activation = trunk_activation
+        self.trunk_last_activation = trunk_last_activation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -59,10 +59,9 @@ def forward(self, inputs):
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
+        y_loc = self.trunk(x_loc)
         if self.trunk_activation:
-            y_loc = self.activation_trunk(self.trunk(x_loc))
-        else:
-            y_loc = self.trunk(x_loc)
+            y_loc = self.activation_trunk(y_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(
@@ -94,7 +93,7 @@ def __init__(
         kernel_initializer,
         layer_sizes_trunk=None,
         regularization=None,
-        trunk_activaiton=False,
+        trunk_last_activation=False,
     ):
         super().__init__()
 
@@ -130,7 +129,7 @@ def __init__(
             )
             self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
-        self.trunk_activation = trunk_activaiton
+        self.trunk_last_activation = trunk_last_activation
 
     def forward(self, inputs):
         x_func1 = inputs[0]

From 16ca4409bbb32716a9d38fb0ef9d5cc902e401dd Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 21:15:00 -0400
Subject: [PATCH 071/170] fix problems

---
 deepxde/nn/pytorch/mionet.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index da5858cc6..723792e61 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -148,10 +148,9 @@ def forward(self, inputs):
             # POD only
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
+            y_loc = self.trunk(x_loc)
             if self.trunk_activation:
-                y_loc = self.activation_trunk(self.trunk(x_loc))
-            else:
-                y_loc = self.trunk(x_loc)
+                y_loc = self.activation_trunk(y_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b
         if self._output_transform is not None:

From 7fafb08402d69fd3b3c745491a5e4a52c178923c Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 22:51:20 -0400
Subject: [PATCH 072/170] fix problem in pytorch/mionet.py

---
 deepxde/nn/pytorch/mionet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 723792e61..69e4967b3 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -60,7 +60,7 @@ def forward(self, inputs):
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
         y_loc = self.trunk(x_loc)
-        if self.trunk_activation:
+        if self.trunk_last_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
@@ -149,7 +149,7 @@ def forward(self, inputs):
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
             y_loc = self.trunk(x_loc)
-            if self.trunk_activation:
+            if self.trunk_last_activation:
                 y_loc = self.activation_trunk(y_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b

From 8ae11fd6cd68c41da3353c21c1ab3d81010b47d1 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 22:54:15 -0400
Subject: [PATCH 073/170] fix problem in pytorch/mionet.py

---
 deepxde/nn/pytorch/mionet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 69e4967b3..723792e61 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -60,7 +60,7 @@ def forward(self, inputs):
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
         y_loc = self.trunk(x_loc)
-        if self.trunk_last_activation:
+        if self.trunk_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
@@ -149,7 +149,7 @@ def forward(self, inputs):
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
             y_loc = self.trunk(x_loc)
-            if self.trunk_last_activation:
+            if self.trunk_activation:
                 y_loc = self.activation_trunk(y_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b

From 4537214d1c82b3b48cd3f8bcf59d93e584693d24 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 28 Jun 2022 22:55:51 -0400
Subject: [PATCH 074/170] fix problem in pytorch/mionet.py

---
 deepxde/nn/pytorch/mionet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 723792e61..69e4967b3 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -60,7 +60,7 @@ def forward(self, inputs):
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
         y_loc = self.trunk(x_loc)
-        if self.trunk_activation:
+        if self.trunk_last_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
         if y_func1.shape[-1] != y_func2.shape[-1]:
@@ -149,7 +149,7 @@ def forward(self, inputs):
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)
         else:
             y_loc = self.trunk(x_loc)
-            if self.trunk_activation:
+            if self.trunk_last_activation:
                 y_loc = self.activation_trunk(y_loc)
             y = torch.einsum("bi,ni->bn", y_func, torch.cat((self.pod_basis, y_loc), 1))
             y += self.b

From 96b28577a19b3d10292f9ae07f22ac000daf4f61 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 18:06:24 -0400
Subject: [PATCH 075/170] pytorch backend PODMIONet add branch connect method
 of "sum","mul","cat" and optional port net after connection.

---
 deepxde/nn/pytorch/mionet.py | 41 ++++++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 69e4967b3..11f5ac0bb 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -89,11 +89,14 @@ def __init__(
         pod_basis,
         layer_sizes_branch1,
         layer_sizes_branch2,
+        layer_sizes_port,
         activation,
         kernel_initializer,
+        connect_method="mul",
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
+        port_net_after_branch=False
     ):
         super().__init__()
 
@@ -101,6 +104,7 @@ def __init__(
             self.activation_branch1 = activations.get(activation["branch1"])
             self.activation_branch2 = activations.get(activation["branch2"])
             self.activation_trunk = activations.get(activation["trunk"])
+            self.activation_port = activations.get(activation["port"])
         else:
             self.activation_branch1 = (
                 self.activation_branch2
@@ -122,6 +126,16 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
+        self.port_net_after_branch = port_net_after_branch
+        if self.port_net_after_branch is True:
+            if callable(layer_sizes_port[1]):
+                # User-defined network
+                self.port = layer_sizes_port[1]
+            else:
+                # Fully connected network
+                self.port = FNN(
+                    layer_sizes_port, self.activation_port, kernel_initializer
+                )
         self.trunk = None
         if layer_sizes_trunk is not None:
             self.trunk = FNN(
@@ -130,6 +144,7 @@ def __init__(
             self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
+        self.connect_method = connect_method
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -138,12 +153,26 @@ def forward(self, inputs):
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
-        # Dot product
-        if y_func1.shape[-1] != y_func2.shape[-1]:
-            raise AssertionError(
-                "Output sizes of branch1 net and branch2 net do not match."
-            )
-        y_func = torch.mul(y_func1, y_func2)
+        # connect two branch outputs
+        if self.connect_method =="cat":
+        	x_port = torch.cat((y_func1,y_func2),1)
+        else:
+            if y_func1.shape[-1] != y_func2.shape[-1]:
+                raise AssertionError(
+                    "Output sizes of branch1 net and branch2 net do not match."
+                )
+            if self.connect_method == "sum":
+                x_port = y_func1+y_func2
+            elif self.connect_method =="mul":
+                x_port = torch.mul(y_func1,y_func2)
+            else:
+                raise NotImplimentedError(f"{self.connect_method} method to be implimented")
+
+        if self.port_net_after_branch is True:
+            y_func = self.port(x_port)
+        else:
+            y_func = x_port
+        #Dot product
         if self.trunk is None:
             # POD only
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)

From 5ad1eeb20c101ffc43032d1311ae66290fde29c2 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 18:28:27 -0400
Subject: [PATCH 076/170] change port net-->merge net

---
 deepxde/nn/pytorch/mionet.py | 42 +++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 11f5ac0bb..f4bd03fd7 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -89,14 +89,14 @@ def __init__(
         pod_basis,
         layer_sizes_branch1,
         layer_sizes_branch2,
-        layer_sizes_port,
         activation,
         kernel_initializer,
         connect_method="mul",
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
-        port_net_after_branch=False
+        layer_sizes_merge=None,
+        merge_net_after_branch=False,
     ):
         super().__init__()
 
@@ -104,7 +104,7 @@ def __init__(
             self.activation_branch1 = activations.get(activation["branch1"])
             self.activation_branch2 = activations.get(activation["branch2"])
             self.activation_trunk = activations.get(activation["trunk"])
-            self.activation_port = activations.get(activation["port"])
+            self.activation_merge = activations.get(activation["merge"])
         else:
             self.activation_branch1 = (
                 self.activation_branch2
@@ -126,15 +126,15 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
-        self.port_net_after_branch = port_net_after_branch
-        if self.port_net_after_branch is True:
-            if callable(layer_sizes_port[1]):
+        self.merge_net_after_branch = merge_net_after_branch
+        if self.merge_net_after_branch is True:
+            if callable(layer_sizes_merge[1]):
                 # User-defined network
-                self.port = layer_sizes_port[1]
+                self.merge = layer_sizes_merge[1]
             else:
                 # Fully connected network
-                self.port = FNN(
-                    layer_sizes_port, self.activation_port, kernel_initializer
+                self.merge = FNN(
+                    layer_sizes_merge, self.activation_merge, kernel_initializer
                 )
         self.trunk = None
         if layer_sizes_trunk is not None:
@@ -154,25 +154,27 @@ def forward(self, inputs):
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
         # connect two branch outputs
-        if self.connect_method =="cat":
-        	x_port = torch.cat((y_func1,y_func2),1)
+        if self.connect_method == "cat":
+            x_merge = torch.cat((y_func1, y_func2), 1)
         else:
             if y_func1.shape[-1] != y_func2.shape[-1]:
                 raise AssertionError(
                     "Output sizes of branch1 net and branch2 net do not match."
                 )
             if self.connect_method == "sum":
-                x_port = y_func1+y_func2
-            elif self.connect_method =="mul":
-                x_port = torch.mul(y_func1,y_func2)
+                x_merge = y_func1 + y_func2
+            elif self.connect_method == "mul":
+                x_merge = torch.mul(y_func1, y_func2)
             else:
-                raise NotImplimentedError(f"{self.connect_method} method to be implimented")
-
-        if self.port_net_after_branch is True:
-            y_func = self.port(x_port)
+                raise NotImplementedError(
+                    f"{self.connect_method} method to be implimented"
+                )
+        # Optional merge net
+        if self.merge_net_after_branch is True:
+            y_func = self.merge(x_merge)
         else:
-            y_func = x_port
-        #Dot product
+            y_func = x_merge
+        # Dot product
         if self.trunk is None:
             # POD only
             y = torch.einsum("bi,ni->bn", y_func, self.pod_basis)

From bf826c6e1ca1e4ddd7c187746b919880f929acbe Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 18:38:58 -0400
Subject: [PATCH 077/170] change merge net-->merger net

---
 deepxde/nn/pytorch/mionet.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index f4bd03fd7..be124b03f 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -95,8 +95,7 @@ def __init__(
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
-        layer_sizes_merge=None,
-        merge_net_after_branch=False,
+        layer_sizes_merger=None
     ):
         super().__init__()
 
@@ -104,7 +103,7 @@ def __init__(
             self.activation_branch1 = activations.get(activation["branch1"])
             self.activation_branch2 = activations.get(activation["branch2"])
             self.activation_trunk = activations.get(activation["trunk"])
-            self.activation_merge = activations.get(activation["merge"])
+            self.activation_merger = activations.get(activation["merger"])
         else:
             self.activation_branch1 = (
                 self.activation_branch2
@@ -126,16 +125,17 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
-        self.merge_net_after_branch = merge_net_after_branch
-        if self.merge_net_after_branch is True:
-            if callable(layer_sizes_merge[1]):
+        if layer_sizes_merger is not None:
+            if callable(layer_sizes_merger[1]):
                 # User-defined network
-                self.merge = layer_sizes_merge[1]
+                self.merger = layer_sizes_merger[1]
             else:
                 # Fully connected network
-                self.merge = FNN(
-                    layer_sizes_merge, self.activation_merge, kernel_initializer
+                self.merger = FNN(
+                    layer_sizes_merger, self.activation_merger, kernel_initializer
                 )
+        else:
+            self.merger = None
         self.trunk = None
         if layer_sizes_trunk is not None:
             self.trunk = FNN(
@@ -155,25 +155,25 @@ def forward(self, inputs):
         y_func2 = self.branch2(x_func2)
         # connect two branch outputs
         if self.connect_method == "cat":
-            x_merge = torch.cat((y_func1, y_func2), 1)
+            x_merger = torch.cat((y_func1, y_func2), 1)
         else:
             if y_func1.shape[-1] != y_func2.shape[-1]:
                 raise AssertionError(
                     "Output sizes of branch1 net and branch2 net do not match."
                 )
             if self.connect_method == "sum":
-                x_merge = y_func1 + y_func2
+                x_merger = y_func1 + y_func2
             elif self.connect_method == "mul":
-                x_merge = torch.mul(y_func1, y_func2)
+                x_merger = torch.mul(y_func1, y_func2)
             else:
                 raise NotImplementedError(
                     f"{self.connect_method} method to be implimented"
                 )
-        # Optional merge net
-        if self.merge_net_after_branch is True:
-            y_func = self.merge(x_merge)
+        # Optional merger net
+        if self.merger is not None:
+            y_func = self.merger(x_merger)
         else:
-            y_func = x_merge
+            y_func = x_merger
         # Dot product
         if self.trunk is None:
             # POD only

From e397040f6bb85d3a54c63f4e787025fb51f2c3d9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 18:40:56 -0400
Subject: [PATCH 078/170] change connect_method-->merge_operation

---
 deepxde/nn/pytorch/mionet.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index be124b03f..9a31409c8 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -91,7 +91,7 @@ def __init__(
         layer_sizes_branch2,
         activation,
         kernel_initializer,
-        connect_method="mul",
+        merge_operation="mul",
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
@@ -144,7 +144,7 @@ def __init__(
             self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
-        self.connect_method = connect_method
+        self.merge_operation = merge_operation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -154,20 +154,20 @@ def forward(self, inputs):
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
         # connect two branch outputs
-        if self.connect_method == "cat":
+        if self.merge_operation == "cat":
             x_merger = torch.cat((y_func1, y_func2), 1)
         else:
             if y_func1.shape[-1] != y_func2.shape[-1]:
                 raise AssertionError(
                     "Output sizes of branch1 net and branch2 net do not match."
                 )
-            if self.connect_method == "sum":
+            if self.merge_operation == "sum":
                 x_merger = y_func1 + y_func2
-            elif self.connect_method == "mul":
+            elif self.merge_operation == "mul":
                 x_merger = torch.mul(y_func1, y_func2)
             else:
                 raise NotImplementedError(
-                    f"{self.connect_method} method to be implimented"
+                    f"{self.merge_operation} operation to be implimented"
                 )
         # Optional merger net
         if self.merger is not None:

From 94017e2e5864a513489fa1afc2989369be1a7ba6 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 18:52:52 -0400
Subject: [PATCH 079/170] black reformat

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 9a31409c8..25bc61c4c 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -95,7 +95,7 @@ def __init__(
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
-        layer_sizes_merger=None
+        layer_sizes_merger=None,
     ):
         super().__init__()
 

From e2be87c7a23309b0d9b442fc45698ba3fe681b2b Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 31 Jul 2022 19:02:37 -0400
Subject: [PATCH 080/170] move merge_operation="mul" to second last

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 25bc61c4c..a110524f1 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -91,10 +91,10 @@ def __init__(
         layer_sizes_branch2,
         activation,
         kernel_initializer,
-        merge_operation="mul",
         layer_sizes_trunk=None,
         regularization=None,
         trunk_last_activation=False,
+        merge_operation="mul",
         layer_sizes_merger=None,
     ):
         super().__init__()

From a93fdcd04af1cc8c81712504b83b70d43d9dcf95 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 14:25:39 -0400
Subject: [PATCH 081/170] add merger net and output merger net in
 MIONetCartesianProd and add monitor in ModelCheckpoint

---
 deepxde/callbacks.py         | 23 ++++++++--
 deepxde/nn/pytorch/mionet.py | 81 ++++++++++++++++++++++++++++++++----
 2 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/deepxde/callbacks.py b/deepxde/callbacks.py
index f3c4b7823..c19aecce2 100644
--- a/deepxde/callbacks.py
+++ b/deepxde/callbacks.py
@@ -119,14 +119,21 @@ class ModelCheckpoint(Callback):
         period: Interval (number of epochs) between checkpoints.
     """
 
-    def __init__(self, filepath, verbose=0, save_better_only=False, period=1):
+    def __init__(
+        self,
+        filepath,
+        verbose=0,
+        save_better_only=False,
+        period=1,
+        monitor="train loss",
+    ):
         super().__init__()
         self.filepath = filepath
         self.verbose = verbose
         self.save_better_only = save_better_only
         self.period = period
 
-        self.monitor = "train loss"
+        self.monitor = monitor
         self.monitor_op = np.less
         self.epochs_since_last_save = 0
         self.best = np.Inf
@@ -137,7 +144,7 @@ def on_epoch_end(self):
             return
         self.epochs_since_last_save = 0
         if self.save_better_only:
-            current = self.model.train_state.best_loss_train
+            current = self.get_monitor_value()
             if self.monitor_op(current, self.best):
                 save_path = self.model.save(self.filepath, verbose=0)
                 if self.verbose > 0:
@@ -154,6 +161,16 @@ def on_epoch_end(self):
         else:
             self.model.save(self.filepath, verbose=self.verbose)
 
+    def get_monitor_value(self):
+        if self.monitor == "train loss":
+            result = sum(self.model.train_state.best_loss_train)
+        elif self.monitor == "test loss":
+            result = sum(self.model.train_state.best_loss_test)
+        else:
+            raise ValueError("The specified monitor function is incorrect.")
+
+        return result
+
 
 class EarlyStopping(Callback):
     """Stop training when a monitored quantity (training or testing loss) has stopped improving.
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index a110524f1..0073289ad 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -17,6 +17,10 @@ def __init__(
         kernel_initializer,
         regularization=None,
         trunk_last_activation=False,
+        merge_operation="mul",
+        layer_sizes_merger=None,
+        output_merge_operation="mul",
+        layer_sizes_output_merger=None,
     ):
         super().__init__()
 
@@ -44,10 +48,38 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
+        if layer_sizes_merger is not None:
+            self.activation_merger = activations.get(activation["merger"])
+            if callable(layer_sizes_merger[1]):
+                # User-defined network
+                self.merger = layer_sizes_merger[1]
+            else:
+                # Fully connected network
+                self.merger = FNN(
+                    layer_sizes_merger, self.activation_merger, kernel_initializer
+                )
+        else:
+            self.merger = None
+        if layer_sizes_output_merger is not None:
+            self.activation_output_merger = activations.get(activation["output merger"])
+            if callable(layer_sizes_output_merger[1]):
+                # User-defined network
+                self.output_merger = layer_sizes_output_merger[1]
+            else:
+                # Fully connected network
+                self.output_merger = FNN(
+                    layer_sizes_output_merger,
+                    self.activation_output_merger,
+                    kernel_initializer,
+                )
+        else:
+            self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
+        self.merge_operation = merge_operation
+        self.output_merge_operation = output_merge_operation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -56,6 +88,26 @@ def forward(self, inputs):
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
+        if self.merge_operation == "cat":
+            x_merger = torch.cat((y_func1, y_func2), 1)
+        else:
+            if y_func1.shape[-1] != y_func2.shape[-1]:
+                raise AssertionError(
+                    "Output sizes of branch1 net and branch2 net do not match."
+                )
+            if self.merge_operation == "sum":
+                x_merger = y_func1 + y_func2
+            elif self.merge_operation == "mul":
+                x_merger = torch.mul(y_func1, y_func2)
+            else:
+                raise NotImplementedError(
+                    f"{self.merge_operation} operation to be implimented"
+                )
+        # Optional merger net
+        if self.merger is not None:
+            y_func = self.merger(x_merger)
+        else:
+            y_func = x_merger
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
@@ -63,16 +115,29 @@ def forward(self, inputs):
         if self.trunk_last_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
-        if y_func1.shape[-1] != y_func2.shape[-1]:
+        if y_func.shape[-1] != y_loc.shape[-1]:
             raise AssertionError(
-                "Output sizes of branch net1 and branch net2 do not match."
+                "Output sizes of merger net and trunk net do not match."
             )
-        y = torch.mul(y_func1, y_func2)
-        if y.shape[-1] != y_loc.shape[-1]:
-            raise AssertionError(
-                "Output sizes of branch net and trunk net do not match."
-            )
-        y = torch.einsum("ip,jp->ij", y, y_loc)
+        # output merger net
+        if self.output_merger is None:
+            y = torch.einsum("ip,jp->ij", y_func, y_loc)
+        else:
+            y_func = y_func[:, None, :]
+            y_loc = y_loc[None, :]
+            if self.output_merge_operation == "mul":
+                y = torch.mul(y_func, y_loc)
+            elif self.output_merge_operation == "sum":
+                y = y_func + yloc
+            elif self.output_merge_operation == "cat":
+                y_func = y_func.repeat(1, y_loc.shape[1], 1)
+                y_loc = y_loc.repeat(y_func.shape[0], 1, 1)
+                y = torch.cat((y_func, y_loc), dim=2)
+            shape0 = y.shape[0]
+            shape1 = y.shape[1]
+            y = y.reshape(shape0 * shape1, -1)
+            y = self.output_merger(y)
+            y.reshape(shape0, shape1)
         # Add bias
         y += self.b
         if self._output_transform is not None:

From d2f34d0c87865982243b7e481c9f78e379d9d0d1 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 14:38:09 -0400
Subject: [PATCH 082/170] fix problems in callbacks.py

---
 deepxde/callbacks.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/deepxde/callbacks.py b/deepxde/callbacks.py
index c19aecce2..64f3a39f2 100644
--- a/deepxde/callbacks.py
+++ b/deepxde/callbacks.py
@@ -117,6 +117,7 @@ class ModelCheckpoint(Callback):
             monitored. Model is only checked at validation step according to
             ``display_every`` in ``Model.train``.
         period: Interval (number of epochs) between checkpoints.
+        monitor: The loss function that is monitored. Either 'train loss' or 'test loss'
     """
 
     def __init__(
@@ -163,9 +164,9 @@ def on_epoch_end(self):
 
     def get_monitor_value(self):
         if self.monitor == "train loss":
-            result = sum(self.model.train_state.best_loss_train)
+            result = sum(self.model.train_state.loss_train)
         elif self.monitor == "test loss":
-            result = sum(self.model.train_state.best_loss_test)
+            result = sum(self.model.train_state.loss_test)
         else:
             raise ValueError("The specified monitor function is incorrect.")
 

From 3fc4b1e6e0e67e0abba5e3c362fb4561447c675e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 14:41:48 -0400
Subject: [PATCH 083/170] recover mionet.py

---
 deepxde/nn/pytorch/mionet.py | 83 ++++--------------------------------
 1 file changed, 9 insertions(+), 74 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 0073289ad..ff16b2ca5 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -17,10 +17,6 @@ def __init__(
         kernel_initializer,
         regularization=None,
         trunk_last_activation=False,
-        merge_operation="mul",
-        layer_sizes_merger=None,
-        output_merge_operation="mul",
-        layer_sizes_output_merger=None,
     ):
         super().__init__()
 
@@ -48,38 +44,10 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
-        if layer_sizes_merger is not None:
-            self.activation_merger = activations.get(activation["merger"])
-            if callable(layer_sizes_merger[1]):
-                # User-defined network
-                self.merger = layer_sizes_merger[1]
-            else:
-                # Fully connected network
-                self.merger = FNN(
-                    layer_sizes_merger, self.activation_merger, kernel_initializer
-                )
-        else:
-            self.merger = None
-        if layer_sizes_output_merger is not None:
-            self.activation_output_merger = activations.get(activation["output merger"])
-            if callable(layer_sizes_output_merger[1]):
-                # User-defined network
-                self.output_merger = layer_sizes_output_merger[1]
-            else:
-                # Fully connected network
-                self.output_merger = FNN(
-                    layer_sizes_output_merger,
-                    self.activation_output_merger,
-                    kernel_initializer,
-                )
-        else:
-            self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
-        self.merge_operation = merge_operation
-        self.output_merge_operation = output_merge_operation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -88,26 +56,6 @@ def forward(self, inputs):
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
-        if self.merge_operation == "cat":
-            x_merger = torch.cat((y_func1, y_func2), 1)
-        else:
-            if y_func1.shape[-1] != y_func2.shape[-1]:
-                raise AssertionError(
-                    "Output sizes of branch1 net and branch2 net do not match."
-                )
-            if self.merge_operation == "sum":
-                x_merger = y_func1 + y_func2
-            elif self.merge_operation == "mul":
-                x_merger = torch.mul(y_func1, y_func2)
-            else:
-                raise NotImplementedError(
-                    f"{self.merge_operation} operation to be implimented"
-                )
-        # Optional merger net
-        if self.merger is not None:
-            y_func = self.merger(x_merger)
-        else:
-            y_func = x_merger
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
@@ -115,29 +63,16 @@ def forward(self, inputs):
         if self.trunk_last_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
-        if y_func.shape[-1] != y_loc.shape[-1]:
+        if y_func1.shape[-1] != y_func2.shape[-1]:
             raise AssertionError(
-                "Output sizes of merger net and trunk net do not match."
+                "Output sizes of branch net1 and branch net2 do not match."
             )
-        # output merger net
-        if self.output_merger is None:
-            y = torch.einsum("ip,jp->ij", y_func, y_loc)
-        else:
-            y_func = y_func[:, None, :]
-            y_loc = y_loc[None, :]
-            if self.output_merge_operation == "mul":
-                y = torch.mul(y_func, y_loc)
-            elif self.output_merge_operation == "sum":
-                y = y_func + yloc
-            elif self.output_merge_operation == "cat":
-                y_func = y_func.repeat(1, y_loc.shape[1], 1)
-                y_loc = y_loc.repeat(y_func.shape[0], 1, 1)
-                y = torch.cat((y_func, y_loc), dim=2)
-            shape0 = y.shape[0]
-            shape1 = y.shape[1]
-            y = y.reshape(shape0 * shape1, -1)
-            y = self.output_merger(y)
-            y.reshape(shape0, shape1)
+        y = torch.mul(y_func1, y_func2)
+        if y.shape[-1] != y_loc.shape[-1]:
+            raise AssertionError(
+                "Output sizes of branch net and trunk net do not match."
+            )
+        y = torch.einsum("ip,jp->ij", y, y_loc)
         # Add bias
         y += self.b
         if self._output_transform is not None:
@@ -251,4 +186,4 @@ def forward(self, inputs):
             y += self.b
         if self._output_transform is not None:
             y = self._output_transform(inputs, y)
-        return y
+        return y
\ No newline at end of file

From 16443867544ccbcd893c7511a58317e2770e0518 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 14:49:29 -0400
Subject: [PATCH 084/170] fix problems

---
 deepxde/callbacks.py         | 2 +-
 deepxde/nn/pytorch/mionet.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/callbacks.py b/deepxde/callbacks.py
index 64f3a39f2..e5113da5a 100644
--- a/deepxde/callbacks.py
+++ b/deepxde/callbacks.py
@@ -117,7 +117,7 @@ class ModelCheckpoint(Callback):
             monitored. Model is only checked at validation step according to
             ``display_every`` in ``Model.train``.
         period: Interval (number of epochs) between checkpoints.
-        monitor: The loss function that is monitored. Either 'train loss' or 'test loss'
+        monitor: The loss function that is monitored. Either 'train loss' or 'test loss.'
     """
 
     def __init__(
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index ff16b2ca5..a110524f1 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -186,4 +186,4 @@ def forward(self, inputs):
             y += self.b
         if self._output_transform is not None:
             y = self._output_transform(inputs, y)
-        return y
\ No newline at end of file
+        return y

From 59ffb46cf56073ba970a809f57d780e10bffad15 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 14:53:16 -0400
Subject: [PATCH 085/170] fix problem

---
 deepxde/callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/callbacks.py b/deepxde/callbacks.py
index e5113da5a..ed623e5d0 100644
--- a/deepxde/callbacks.py
+++ b/deepxde/callbacks.py
@@ -117,7 +117,7 @@ class ModelCheckpoint(Callback):
             monitored. Model is only checked at validation step according to
             ``display_every`` in ``Model.train``.
         period: Interval (number of epochs) between checkpoints.
-        monitor: The loss function that is monitored. Either 'train loss' or 'test loss.'
+        monitor: The loss function that is monitored. Either 'train loss' or 'test loss'.
     """
 
     def __init__(

From e65b392dad6677d81b70014072d0b7563f4f1e00 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 15:01:14 -0400
Subject: [PATCH 086/170] Add merger net and ouput merger net in
 MIONetCartesianProd

---
 deepxde/nn/pytorch/mionet.py | 81 ++++++++++++++++++++++++++++++++----
 1 file changed, 73 insertions(+), 8 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index a110524f1..0073289ad 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -17,6 +17,10 @@ def __init__(
         kernel_initializer,
         regularization=None,
         trunk_last_activation=False,
+        merge_operation="mul",
+        layer_sizes_merger=None,
+        output_merge_operation="mul",
+        layer_sizes_output_merger=None,
     ):
         super().__init__()
 
@@ -44,10 +48,38 @@ def __init__(
             self.branch2 = FNN(
                 layer_sizes_branch2, self.activation_branch2, kernel_initializer
             )
+        if layer_sizes_merger is not None:
+            self.activation_merger = activations.get(activation["merger"])
+            if callable(layer_sizes_merger[1]):
+                # User-defined network
+                self.merger = layer_sizes_merger[1]
+            else:
+                # Fully connected network
+                self.merger = FNN(
+                    layer_sizes_merger, self.activation_merger, kernel_initializer
+                )
+        else:
+            self.merger = None
+        if layer_sizes_output_merger is not None:
+            self.activation_output_merger = activations.get(activation["output merger"])
+            if callable(layer_sizes_output_merger[1]):
+                # User-defined network
+                self.output_merger = layer_sizes_output_merger[1]
+            else:
+                # Fully connected network
+                self.output_merger = FNN(
+                    layer_sizes_output_merger,
+                    self.activation_output_merger,
+                    kernel_initializer,
+                )
+        else:
+            self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
         self.b = torch.tensor(0.0, requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
+        self.merge_operation = merge_operation
+        self.output_merge_operation = output_merge_operation
 
     def forward(self, inputs):
         x_func1 = inputs[0]
@@ -56,6 +88,26 @@ def forward(self, inputs):
         # Branch net to encode the input function
         y_func1 = self.branch1(x_func1)
         y_func2 = self.branch2(x_func2)
+        if self.merge_operation == "cat":
+            x_merger = torch.cat((y_func1, y_func2), 1)
+        else:
+            if y_func1.shape[-1] != y_func2.shape[-1]:
+                raise AssertionError(
+                    "Output sizes of branch1 net and branch2 net do not match."
+                )
+            if self.merge_operation == "sum":
+                x_merger = y_func1 + y_func2
+            elif self.merge_operation == "mul":
+                x_merger = torch.mul(y_func1, y_func2)
+            else:
+                raise NotImplementedError(
+                    f"{self.merge_operation} operation to be implimented"
+                )
+        # Optional merger net
+        if self.merger is not None:
+            y_func = self.merger(x_merger)
+        else:
+            y_func = x_merger
         # Trunk net to encode the domain of the output function
         if self._input_transform is not None:
             y_loc = self._input_transform(x_loc)
@@ -63,16 +115,29 @@ def forward(self, inputs):
         if self.trunk_last_activation:
             y_loc = self.activation_trunk(y_loc)
         # Dot product
-        if y_func1.shape[-1] != y_func2.shape[-1]:
+        if y_func.shape[-1] != y_loc.shape[-1]:
             raise AssertionError(
-                "Output sizes of branch net1 and branch net2 do not match."
+                "Output sizes of merger net and trunk net do not match."
             )
-        y = torch.mul(y_func1, y_func2)
-        if y.shape[-1] != y_loc.shape[-1]:
-            raise AssertionError(
-                "Output sizes of branch net and trunk net do not match."
-            )
-        y = torch.einsum("ip,jp->ij", y, y_loc)
+        # output merger net
+        if self.output_merger is None:
+            y = torch.einsum("ip,jp->ij", y_func, y_loc)
+        else:
+            y_func = y_func[:, None, :]
+            y_loc = y_loc[None, :]
+            if self.output_merge_operation == "mul":
+                y = torch.mul(y_func, y_loc)
+            elif self.output_merge_operation == "sum":
+                y = y_func + yloc
+            elif self.output_merge_operation == "cat":
+                y_func = y_func.repeat(1, y_loc.shape[1], 1)
+                y_loc = y_loc.repeat(y_func.shape[0], 1, 1)
+                y = torch.cat((y_func, y_loc), dim=2)
+            shape0 = y.shape[0]
+            shape1 = y.shape[1]
+            y = y.reshape(shape0 * shape1, -1)
+            y = self.output_merger(y)
+            y.reshape(shape0, shape1)
         # Add bias
         y += self.b
         if self._output_transform is not None:

From 8ddb300cc730de18cfde11ff397b64130c977e64 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 15 Aug 2022 15:10:43 -0400
Subject: [PATCH 087/170] fix problem

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 0073289ad..2e679ff42 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -128,7 +128,7 @@ def forward(self, inputs):
             if self.output_merge_operation == "mul":
                 y = torch.mul(y_func, y_loc)
             elif self.output_merge_operation == "sum":
-                y = y_func + yloc
+                y = y_func + y_loc
             elif self.output_merge_operation == "cat":
                 y_func = y_func.repeat(1, y_loc.shape[1], 1)
                 y_loc = y_loc.repeat(y_func.shape[0], 1, 1)

From 4b5f322e1114ba745498e6a47468062cd8d87fbc Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 25 Aug 2022 21:15:09 -0400
Subject: [PATCH 088/170] change sum operation to add operation

---
 deepxde/nn/pytorch/mionet.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index 2e679ff42..f81928e36 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -95,7 +95,7 @@ def forward(self, inputs):
                 raise AssertionError(
                     "Output sizes of branch1 net and branch2 net do not match."
                 )
-            if self.merge_operation == "sum":
+            if self.merge_operation == "add":
                 x_merger = y_func1 + y_func2
             elif self.merge_operation == "mul":
                 x_merger = torch.mul(y_func1, y_func2)
@@ -127,7 +127,7 @@ def forward(self, inputs):
             y_loc = y_loc[None, :]
             if self.output_merge_operation == "mul":
                 y = torch.mul(y_func, y_loc)
-            elif self.output_merge_operation == "sum":
+            elif self.output_merge_operation == "add":
                 y = y_func + y_loc
             elif self.output_merge_operation == "cat":
                 y_func = y_func.repeat(1, y_loc.shape[1], 1)
@@ -226,7 +226,7 @@ def forward(self, inputs):
                 raise AssertionError(
                     "Output sizes of branch1 net and branch2 net do not match."
                 )
-            if self.merge_operation == "sum":
+            if self.merge_operation == "add":
                 x_merger = y_func1 + y_func2
             elif self.merge_operation == "mul":
                 x_merger = torch.mul(y_func1, y_func2)

From bc4304f55859e468744e65e35b0f251265e267e4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 9 Sep 2022 21:22:56 -0400
Subject: [PATCH 089/170] Fix bug in pytorch backended mionet

---
 deepxde/nn/pytorch/mionet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index f81928e36..ff0e8faf3 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -137,7 +137,7 @@ def forward(self, inputs):
             shape1 = y.shape[1]
             y = y.reshape(shape0 * shape1, -1)
             y = self.output_merger(y)
-            y.reshape(shape0, shape1)
+            y = y.reshape(shape0, shape1)
         # Add bias
         y += self.b
         if self._output_transform is not None:

From 482f40fdc95f4ca3dd3e1a1dacd7c164181ee2fa Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 20:56:32 -0400
Subject: [PATCH 090/170] add minibatch

---
 deepxde/data/quadruple.py | 11 +++++++----
 deepxde/data/triple.py    |  9 ++++++---
 deepxde/model.py          | 12 +++++++-----
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index 520ec8555..af4301bde 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -25,10 +25,11 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None):
+    def train_next_batch(self, batch_size=None, minibatch=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
+        # TODO: minibatch
         return (
             (self.train_x[0][indices], self.train_x[1][indices]),
             self.train_x[2][indices],
@@ -73,19 +74,21 @@ def __init__(self, X_train, y_train, X_test, y_test):
         self.test_x, self.test_y = X_test, y_test
 
         self.train_sampler = BatchSampler(len(X_train[0]), shuffle=True)
+        self.trunk_sampler = BatchSampler(len(X_train[2]), shuffle=True)
 
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None):
+    def train_next_batch(self, batch_size=None,minibatch=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
+        minibatch_indices = self.trunk_sampler.get_next(minibatch)
         return (
             self.train_x[0][indices],
             self.train_x[1][indices],
-            self.train_x[2],
-        ), self.train_y[indices]
+            self.train_x[2][minibatch_indices],
+        ), self.train_y[indices,minibatch_indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index ce3f182ae..2d82fe952 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -31,10 +31,11 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None):
+    def train_next_batch(self, batch_size=None, minibatch=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
+        # TODO: minibatch
         return (
             (self.train_x[0][indices], self.train_x[1][indices]),
             self.train_y[indices],
@@ -72,15 +73,17 @@ def __init__(self, X_train, y_train, X_test, y_test):
         self.test_x, self.test_y = X_test, y_test
 
         self.train_sampler = BatchSampler(len(X_train[0]), shuffle=True)
+        self.trunk_sampler = BatchSampler(len(X_train[2]), shuffle=True)
 
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None):
+    def train_next_batch(self, batch_size=None, minibatch=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
-        return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
+        minibatch_indices = self.trunk_sampler.get_next(minibatch)
+        return (self.train_x[0][indices], self.train_x[1][minibatch_indices]), self.train_y[indices,minibatch_indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/model.py b/deepxde/model.py
index 334a34dcf..a3e908d56 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -521,6 +521,7 @@ def train(
         self,
         iterations=None,
         batch_size=None,
+        minibatch=None,
         display_every=1000,
         disregard_previous_best=False,
         callbacks=None,
@@ -555,6 +556,7 @@ def train(
             )
             iterations = epochs
         self.batch_size = batch_size
+        self.minibatch = minibatch
         self.callbacks = CallbackList(callbacks=callbacks)
         self.callbacks.set_model(self)
         if disregard_previous_best:
@@ -572,7 +574,7 @@ def train(
 
         print("Training model...\n")
         self.stop_training = False
-        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size))
+        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size,self.minibatch))
         self.train_state.set_data_test(*self.data.test())
         self._test()
         self.callbacks.on_train_begin()
@@ -601,7 +603,7 @@ def _train_sgd(self, iterations, display_every):
             self.callbacks.on_batch_begin()
 
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size)
+                *self.data.train_next_batch(self.batch_size,self.minibatch)
             )
             self._train_step(
                 self.train_state.X_train,
@@ -651,7 +653,7 @@ def loss_callback(loss_train, loss_test, *args):
                         )
                         cb.file.flush()
 
-        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size))
+        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size,self.minibatch))
         feed_dict = self.net.feed_dict(
             True,
             self.train_state.X_train,
@@ -677,7 +679,7 @@ def _train_tensorflow_tfp(self):
         n_iter = 0
         while n_iter < optimizers.LBFGS_options["maxiter"]:
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size)
+                *self.data.train_next_batch(self.batch_size,self.minibatch)
             )
             results = self.train_step(
                 self.train_state.X_train,
@@ -699,7 +701,7 @@ def _train_pytorch_lbfgs(self):
             self.callbacks.on_batch_begin()
 
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size)
+                *self.data.train_next_batch(self.batch_size,self.minibatch)
             )
             self._train_step(
                 self.train_state.X_train,

From 72f412badb21affa525f7ad755d11bda083fd5ae Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 20:58:32 -0400
Subject: [PATCH 091/170] fix bug

---
 deepxde/data/triple.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index 2d82fe952..0bbb5c223 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -73,7 +73,7 @@ def __init__(self, X_train, y_train, X_test, y_test):
         self.test_x, self.test_y = X_test, y_test
 
         self.train_sampler = BatchSampler(len(X_train[0]), shuffle=True)
-        self.trunk_sampler = BatchSampler(len(X_train[2]), shuffle=True)
+        self.trunk_sampler = BatchSampler(len(X_train[1]), shuffle=True)
 
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)

From affc5ca89bb3b3e8c63261cbfde95aa261daefc4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 22:13:28 -0400
Subject: [PATCH 092/170] add minibatch

---
 deepxde/data/quadruple.py | 18 ++++++++++++------
 deepxde/data/triple.py    | 15 ++++++++++-----
 deepxde/model.py          | 12 +++++-------
 3 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index af4301bde..d4b539c81 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -25,11 +25,10 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None, minibatch=None):
+    def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
-        # TODO: minibatch
         return (
             (self.train_x[0][indices], self.train_x[1][indices]),
             self.train_x[2][indices],
@@ -79,16 +78,23 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None,minibatch=None):
+    def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
+        if len(batch_size) == 2:
+            indices = self.train_sampler.get_next(batch_size[0])
+            minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+            return (
+                       self.train_x[0][indices],
+                       self.train_x[1][indices],
+                       self.train_x[2][minibatch_indices],
+                   ), self.train_y[indices,minibatch_indices]
         indices = self.train_sampler.get_next(batch_size)
-        minibatch_indices = self.trunk_sampler.get_next(minibatch)
         return (
             self.train_x[0][indices],
             self.train_x[1][indices],
-            self.train_x[2][minibatch_indices],
-        ), self.train_y[indices,minibatch_indices]
+            self.train_x[2],
+        ), self.train_y[indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index 0bbb5c223..398830255 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -31,11 +31,10 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None, minibatch=None):
+    def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
         indices = self.train_sampler.get_next(batch_size)
-        # TODO: minibatch
         return (
             (self.train_x[0][indices], self.train_x[1][indices]),
             self.train_y[indices],
@@ -78,12 +77,18 @@ def __init__(self, X_train, y_train, X_test, y_test):
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return loss_fn(targets, outputs)
 
-    def train_next_batch(self, batch_size=None, minibatch=None):
+    def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
+        if len(batch_size) == 2:
+            indices = self.train_sampler.get_next(batch_size[0])
+            minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+            return (
+                self.train_x[0][indices],
+                self.train_x[1][minibatch_indices],
+            ), self.train_y[indices, minibatch_indices]
         indices = self.train_sampler.get_next(batch_size)
-        minibatch_indices = self.trunk_sampler.get_next(minibatch)
-        return (self.train_x[0][indices], self.train_x[1][minibatch_indices]), self.train_y[indices,minibatch_indices]
+        return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/model.py b/deepxde/model.py
index a3e908d56..334a34dcf 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -521,7 +521,6 @@ def train(
         self,
         iterations=None,
         batch_size=None,
-        minibatch=None,
         display_every=1000,
         disregard_previous_best=False,
         callbacks=None,
@@ -556,7 +555,6 @@ def train(
             )
             iterations = epochs
         self.batch_size = batch_size
-        self.minibatch = minibatch
         self.callbacks = CallbackList(callbacks=callbacks)
         self.callbacks.set_model(self)
         if disregard_previous_best:
@@ -574,7 +572,7 @@ def train(
 
         print("Training model...\n")
         self.stop_training = False
-        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size,self.minibatch))
+        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size))
         self.train_state.set_data_test(*self.data.test())
         self._test()
         self.callbacks.on_train_begin()
@@ -603,7 +601,7 @@ def _train_sgd(self, iterations, display_every):
             self.callbacks.on_batch_begin()
 
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size,self.minibatch)
+                *self.data.train_next_batch(self.batch_size)
             )
             self._train_step(
                 self.train_state.X_train,
@@ -653,7 +651,7 @@ def loss_callback(loss_train, loss_test, *args):
                         )
                         cb.file.flush()
 
-        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size,self.minibatch))
+        self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size))
         feed_dict = self.net.feed_dict(
             True,
             self.train_state.X_train,
@@ -679,7 +677,7 @@ def _train_tensorflow_tfp(self):
         n_iter = 0
         while n_iter < optimizers.LBFGS_options["maxiter"]:
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size,self.minibatch)
+                *self.data.train_next_batch(self.batch_size)
             )
             results = self.train_step(
                 self.train_state.X_train,
@@ -701,7 +699,7 @@ def _train_pytorch_lbfgs(self):
             self.callbacks.on_batch_begin()
 
             self.train_state.set_data_train(
-                *self.data.train_next_batch(self.batch_size,self.minibatch)
+                *self.data.train_next_batch(self.batch_size)
             )
             self._train_step(
                 self.train_state.X_train,

From 654404806f92f51994365db96cc37e4aec3a5200 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 22:30:46 -0400
Subject: [PATCH 093/170] fix bug

---
 deepxde/data/quadruple.py | 8 ++++----
 deepxde/data/triple.py    | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index d4b539c81..6f4db5464 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -72,7 +72,7 @@ def __init__(self, X_train, y_train, X_test, y_test):
         self.train_x, self.train_y = X_train, y_train
         self.test_x, self.test_y = X_test, y_test
 
-        self.train_sampler = BatchSampler(len(X_train[0]), shuffle=True)
+        self.branch_sampler = BatchSampler(len(X_train[0]), shuffle=True)
         self.trunk_sampler = BatchSampler(len(X_train[2]), shuffle=True)
 
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
@@ -81,15 +81,15 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if len(batch_size) == 2:
-            indices = self.train_sampler.get_next(batch_size[0])
+        if isinstance(batch_size, tuple) or isinstance(batch_size,list):
+            indices = self.branch_sampler.get_next(batch_size[0])
             minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
             return (
                        self.train_x[0][indices],
                        self.train_x[1][indices],
                        self.train_x[2][minibatch_indices],
                    ), self.train_y[indices,minibatch_indices]
-        indices = self.train_sampler.get_next(batch_size)
+        indices = self.branch_sampler.get_next(batch_size)
         return (
             self.train_x[0][indices],
             self.train_x[1][indices],
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index 398830255..0544d6775 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -71,7 +71,7 @@ def __init__(self, X_train, y_train, X_test, y_test):
         self.train_x, self.train_y = X_train, y_train
         self.test_x, self.test_y = X_test, y_test
 
-        self.train_sampler = BatchSampler(len(X_train[0]), shuffle=True)
+        self.branch_sampler = BatchSampler(len(X_train[0]), shuffle=True)
         self.trunk_sampler = BatchSampler(len(X_train[1]), shuffle=True)
 
     def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
@@ -80,14 +80,14 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if len(batch_size) == 2:
-            indices = self.train_sampler.get_next(batch_size[0])
+        if isinstance(batch_size, tuple) or isinstance(batch_size,list):
+            indices = self.branch_sampler.get_next(batch_size[0])
             minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
             return (
                 self.train_x[0][indices],
                 self.train_x[1][minibatch_indices],
             ), self.train_y[indices, minibatch_indices]
-        indices = self.train_sampler.get_next(batch_size)
+        indices = self.branch_sampler.get_next(batch_size)
         return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
 
     def test(self):

From 94c7f514984ab5ec90ae2c37920032f9f3f58a14 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 22:45:26 -0400
Subject: [PATCH 094/170] update docstring

---
 deepxde/model.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 334a34dcf..6564434be 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -533,10 +533,11 @@ def train(
         Args:
             iterations (Integer): Number of iterations to train the model, i.e., number
                 of times the network weights are updated.
-            batch_size: Integer or ``None``. If you solve PDEs via ``dde.data.PDE`` or
-                ``dde.data.TimePDE``, do not use `batch_size`, and instead use
-                `dde.callbacks.PDEResidualResampler
-                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+            batch_size: Integer, turple, list or ``None``. if you want to use minibatch for trunk net input,
+            set `batch_size` as a turple or list,
+            fist location is the batch size and second location is minibatch size.
+             If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,
+             do not use `batch_size`, and instead use dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best

From 663af65fb2b847842995105d6ab7ff155230f649 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Fri, 21 Oct 2022 23:01:15 -0400
Subject: [PATCH 095/170] fix bug

---
 deepxde/data/quadruple.py | 2 +-
 deepxde/data/triple.py    | 2 +-
 deepxde/model.py          | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index 6f4db5464..7a4a147a7 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -81,7 +81,7 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if isinstance(batch_size, tuple) or isinstance(batch_size,list):
+        if isinstance(batch_size, (tuple,list)):
             indices = self.branch_sampler.get_next(batch_size[0])
             minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
             return (
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index 0544d6775..e194c6ebb 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -80,7 +80,7 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if isinstance(batch_size, tuple) or isinstance(batch_size,list):
+        if isinstance(batch_size, (tuple,list)):
             indices = self.branch_sampler.get_next(batch_size[0])
             minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
             return (
diff --git a/deepxde/model.py b/deepxde/model.py
index 6564434be..9a29a62b1 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,8 +536,8 @@ def train(
             batch_size: Integer, turple, list or ``None``. if you want to use minibatch for trunk net input,
             set `batch_size` as a turple or list,
             fist location is the batch size and second location is minibatch size.
-             If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,
-             do not use `batch_size`, and instead use dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+             If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,
+             and instead use dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best

From 0cd00d062e0781a091090f9b91f1ede5379e6d72 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 12:51:37 -0400
Subject: [PATCH 096/170] update docstring

---
 deepxde/model.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 9a29a62b1..71ef95ad2 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -533,11 +533,10 @@ def train(
         Args:
             iterations (Integer): Number of iterations to train the model, i.e., number
                 of times the network weights are updated.
-            batch_size: Integer, turple, list or ``None``. if you want to use minibatch for trunk net input,
-            set `batch_size` as a turple or list,
-            fist location is the batch size and second location is minibatch size.
-             If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,
-             and instead use dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+            batch_size: Integer, turple, or ``None``. if you want to use minibatch for trunk net input,
+                set `batch_size` as a turple, fist location is the batch size and second location is minibatch size.
+                If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,
+                and instead use `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best

From 35efc65444d671ae9f036038b12a643db236706f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 13:05:08 -0400
Subject: [PATCH 097/170] update

---
 deepxde/data/quadruple.py | 20 ++++++++++----------
 deepxde/data/triple.py    | 18 +++++++++---------
 deepxde/model.py          |  7 +++----
 3 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index 7a4a147a7..9f2ed86de 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -81,20 +81,20 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if isinstance(batch_size, (tuple,list)):
-            indices = self.branch_sampler.get_next(batch_size[0])
-            minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+        if not isinstance(batch_size, (tuple, list)):
+            indices = self.branch_sampler.get_next(batch_size)
             return (
-                       self.train_x[0][indices],
-                       self.train_x[1][indices],
-                       self.train_x[2][minibatch_indices],
-                   ), self.train_y[indices,minibatch_indices]
-        indices = self.branch_sampler.get_next(batch_size)
+                self.train_x[0][indices],
+                self.train_x[1][indices],
+                self.train_x[2],
+            ), self.train_y[indices]
+        indices = self.branch_sampler.get_next(batch_size[0])
+        minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
         return (
             self.train_x[0][indices],
             self.train_x[1][indices],
-            self.train_x[2],
-        ), self.train_y[indices]
+            self.train_x[2][minibatch_indices],
+        ), self.train_y[indices, minibatch_indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index e194c6ebb..054c1760b 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -80,15 +80,15 @@ def losses(self, targets, outputs, loss_fn, inputs, model, aux=None):
     def train_next_batch(self, batch_size=None):
         if batch_size is None:
             return self.train_x, self.train_y
-        if isinstance(batch_size, (tuple,list)):
-            indices = self.branch_sampler.get_next(batch_size[0])
-            minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
-            return (
-                self.train_x[0][indices],
-                self.train_x[1][minibatch_indices],
-            ), self.train_y[indices, minibatch_indices]
-        indices = self.branch_sampler.get_next(batch_size)
-        return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
+        if not isinstance(batch_size, (tuple, list)):
+            indices = self.branch_sampler.get_next(batch_size)
+            return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
+        indices = self.branch_sampler.get_next(batch_size[0])
+        minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+        return (
+            self.train_x[0][indices],
+            self.train_x[1][minibatch_indices],
+        ), self.train_y[indices, minibatch_indices]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/model.py b/deepxde/model.py
index 71ef95ad2..5423cb3c4 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -533,11 +533,10 @@ def train(
         Args:
             iterations (Integer): Number of iterations to train the model, i.e., number
                 of times the network weights are updated.
-            batch_size: Integer, turple, or ``None``. if you want to use minibatch for trunk net input,
-                set `batch_size` as a turple, fist location is the batch size and second location is minibatch size.
+            batch_size: Integer, tuple, or ``None``. if you want to use minibatch for trunk net input,
+                set `batch_size` as a tuple, fist location is the batch size and second location is minibatch size.
                 If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,
-                and instead use `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                and instead use `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From 321b8a9be6151fd6cbea4cc691ab8cba084394b1 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 13:07:16 -0400
Subject: [PATCH 098/170] update docstring

---
 deepxde/model.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 5423cb3c4..3c461d0a4 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -535,8 +535,10 @@ def train(
                 of times the network weights are updated.
             batch_size: Integer, tuple, or ``None``. if you want to use minibatch for trunk net input,
                 set `batch_size` as a tuple, fist location is the batch size and second location is minibatch size.
-                If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,
-                and instead use `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,and instead use
+                `dde.callbacks.PDEResidualResampler
+                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From 6d24a0c22b5a9c4524882765006c99218951fd2d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 13:46:05 -0400
Subject: [PATCH 099/170] update

---
 deepxde/data/quadruple.py | 12 ++++++------
 deepxde/data/triple.py    | 10 +++++-----
 deepxde/model.py          | 13 +++++++------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index 9f2ed86de..2e9fc0ee4 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -88,13 +88,13 @@ def train_next_batch(self, batch_size=None):
                 self.train_x[1][indices],
                 self.train_x[2],
             ), self.train_y[indices]
-        indices = self.branch_sampler.get_next(batch_size[0])
-        minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+        indices_branch = self.branch_sampler.get_next(batch_size[0])
+        indices_trunk = self.trunk_sampler.get_next(batch_size[1])
         return (
-            self.train_x[0][indices],
-            self.train_x[1][indices],
-            self.train_x[2][minibatch_indices],
-        ), self.train_y[indices, minibatch_indices]
+            self.train_x[0][indices_branch],
+            self.train_x[1][indices_branch],
+            self.train_x[2][indices_trunk],
+        ), self.train_y[indices_branch, indices_trunk]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index 054c1760b..ff5ec6bd7 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -83,12 +83,12 @@ def train_next_batch(self, batch_size=None):
         if not isinstance(batch_size, (tuple, list)):
             indices = self.branch_sampler.get_next(batch_size)
             return (self.train_x[0][indices], self.train_x[1]), self.train_y[indices]
-        indices = self.branch_sampler.get_next(batch_size[0])
-        minibatch_indices = self.trunk_sampler.get_next(batch_size[1])
+        indices_branch = self.branch_sampler.get_next(batch_size[0])
+        indices_trunk = self.trunk_sampler.get_next(batch_size[1])
         return (
-            self.train_x[0][indices],
-            self.train_x[1][minibatch_indices],
-        ), self.train_y[indices, minibatch_indices]
+            self.train_x[0][indices_branch],
+            self.train_x[1][indices_trunk],
+        ), self.train_y[indices_branch, indices_trunk]
 
     def test(self):
         return self.test_x, self.test_y
diff --git a/deepxde/model.py b/deepxde/model.py
index 3c461d0a4..d5778a65e 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -533,12 +533,13 @@ def train(
         Args:
             iterations (Integer): Number of iterations to train the model, i.e., number
                 of times the network weights are updated.
-            batch_size: Integer, tuple, or ``None``. if you want to use minibatch for trunk net input,
-                set `batch_size` as a tuple, fist location is the batch size and second location is minibatch size.
-                If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,and instead use
-                `dde.callbacks.PDEResidualResampler
-                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+            batch_size: Integer, tuple, or ``None``.
+                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,and instead use
+                    `dde.callbacks.PDEResidualResampler
+                    <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                - For DeepONet, if you want to use mini-batch for trunk net input,
+                    set `batch_size` as a tuple, fist location is the batch size and second location is mini-batch size.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From 71af04268ce5af02f969bce110cc455ed505e8b0 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 14:37:28 -0400
Subject: [PATCH 100/170] update docstring

---
 deepxde/data/triple.py |  2 +-
 deepxde/model.py       | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index ff5ec6bd7..b9d0f4252 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -55,7 +55,7 @@ class TripleCartesianProd(Data):
     Args:
         X_train: A tuple of two NumPy arrays. The first element has the shape (`N1`,
             `dim1`), and the second element has the shape (`N2`, `dim2`). The mini-batch
-            is only applied to `N1`.
+            is applied to `N1` and `N2`.
         y_train: A NumPy array of shape (`N1`, `N2`).
     """
 
diff --git a/deepxde/model.py b/deepxde/model.py
index d5778a65e..a3cffa0dd 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -534,12 +534,16 @@ def train(
             iterations (Integer): Number of iterations to train the model, i.e., number
                 of times the network weights are updated.
             batch_size: Integer, tuple, or ``None``.
+
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,and instead use
-                    `dde.callbacks.PDEResidualResampler
-                    <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
-                - For DeepONet, if you want to use mini-batch for trunk net input,
-                    set `batch_size` as a tuple, fist location is the batch size and second location is mini-batch size.
+                `dde.callbacks.PDEResidualResampler
+                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+
+                - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
+                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From d6c3cb9d85fefc22dd1e7ebbe1702616baa285e9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 14:38:35 -0400
Subject: [PATCH 101/170] update docstring

---
 deepxde/data/quadruple.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index 2e9fc0ee4..a69cac237 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -48,7 +48,7 @@ class QuadrupleCartesianProd(Data):
     Args:
         X_train: A tuple of three NumPy arrays. The first element has the shape (`N1`,
             `dim1`), the second element has the shape (`N1`, `dim2`), and the third
-            element has the shape (`N2`, `dim3`).
+            element has the shape (`N2`, `dim3`).The mini-batch is applied to `N1` and `N2`.
         y_train: A NumPy array of shape (`N1`, `N2`).
     """
 

From 35158bcaeae0391587532e596da366b32a38dc41 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 14:39:51 -0400
Subject: [PATCH 102/170] update docstring

---
 deepxde/model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index a3cffa0dd..1aa4934bd 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -535,7 +535,7 @@ def train(
                 of times the network weights are updated.
             batch_size: Integer, tuple, or ``None``.
 
-                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``,do not use `batch_size`,and instead use
+                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`,and instead use
                 `dde.callbacks.PDEResidualResampler
                 <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.

From 25426466b0b0aacb69e14386cad0488d8d022cc9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 14:55:27 -0400
Subject: [PATCH 103/170] update docstring

---
 deepxde/model.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 1aa4934bd..2f2953ea9 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,14 +536,14 @@ def train(
             batch_size: Integer, tuple, or ``None``.
 
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`,and instead use
-                `dde.callbacks.PDEResidualResampler
-                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
-
-                - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
-                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
-                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
-                and the second number is the batch size for the trunk net input.
+                    `dde.callbacks.PDEResidualResampler
+                    <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+
+                - For DeepONet in the format of Cartesian product, if ``batch_size`` is an Integer,
+                    then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                    set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                    and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From d743f044604d863c5cf498be79e590a0d884835e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:13:57 -0400
Subject: [PATCH 104/170] update docstring

---
 deepxde/model.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 2f2953ea9..8c7cdb8b0 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,14 +536,13 @@ def train(
             batch_size: Integer, tuple, or ``None``.
 
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`,and instead use
-                    `dde.callbacks.PDEResidualResampler
-                    <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
-
+                `dde.callbacks.PDEResidualResampler
+                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
                 - For DeepONet in the format of Cartesian product, if ``batch_size`` is an Integer,
-                    then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
-                    set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
-                    and the second number is the batch size for the trunk net input.
+                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From 1aee36f21d8a30ffb19079d68d6a2fbad4716269 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:17:30 -0400
Subject: [PATCH 105/170] update docstring

---
 deepxde/data/triple.py | 3 +--
 deepxde/model.py       | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/deepxde/data/triple.py b/deepxde/data/triple.py
index b9d0f4252..9b385d939 100644
--- a/deepxde/data/triple.py
+++ b/deepxde/data/triple.py
@@ -54,8 +54,7 @@ class TripleCartesianProd(Data):
 
     Args:
         X_train: A tuple of two NumPy arrays. The first element has the shape (`N1`,
-            `dim1`), and the second element has the shape (`N2`, `dim2`). The mini-batch
-            is applied to `N1` and `N2`.
+            `dim1`), and the second element has the shape (`N2`, `dim2`).
         y_train: A NumPy array of shape (`N1`, `N2`).
     """
 
diff --git a/deepxde/model.py b/deepxde/model.py
index 8c7cdb8b0..635196cc6 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -535,7 +535,7 @@ def train(
                 of times the network weights are updated.
             batch_size: Integer, tuple, or ``None``.
 
-                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`,and instead use
+                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`, and instead use
                 `dde.callbacks.PDEResidualResampler
                 <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.

From 57a2e55aa8440587a54250983fc074c0daa5acc2 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:26:40 -0400
Subject: [PATCH 106/170] update docstring

---
 deepxde/model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 635196cc6..11f73e6d7 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -539,7 +539,7 @@ def train(
                 `dde.callbacks.PDEResidualResampler
                 <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
-                - For DeepONet in the format of Cartesian product, if ``batch_size`` is an Integer,
+                - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
                 then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
                 set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
                 and the second number is the batch size for the trunk net input.

From b7f07e89a39af3621efb56337ab9884981128768 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:36:33 -0400
Subject: [PATCH 107/170] update docstring

---
 deepxde/model.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/model.py b/deepxde/model.py
index 11f73e6d7..19335a9d7 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -539,6 +539,7 @@ def train(
                 `dde.callbacks.PDEResidualResampler
                 <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                 see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+
                 - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
                 then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
                 set `batch_size` as a tuple, where the fist number is the batch size for the branch net input

From 1f949948ea948d191a2173c50ffceb5465bf8f63 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:46:39 -0400
Subject: [PATCH 108/170] update docstring

---
 deepxde/model.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 19335a9d7..428a41c9c 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,14 +536,13 @@ def train(
             batch_size: Integer, tuple, or ``None``.
 
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`, and instead use
-                `dde.callbacks.PDEResidualResampler
-                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                    `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
 
                 - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
-                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
-                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
-                and the second number is the batch size for the trunk net input.
+                    then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                    set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                    and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From d6b81a1130d683e87f1db1d9d7ac9f3e10f7ca56 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 15:50:54 -0400
Subject: [PATCH 109/170] update docstring

---
 deepxde/model.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 428a41c9c..19335a9d7 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,13 +536,14 @@ def train(
             batch_size: Integer, tuple, or ``None``.
 
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`, and instead use
-                    `dde.callbacks.PDEResidualResampler<https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                    see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                `dde.callbacks.PDEResidualResampler
+                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
 
                 - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
-                    then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
-                    set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
-                    and the second number is the batch size for the trunk net input.
+                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From a8bfe615a8d8ba8a3ae54b2b7bb7832d52caf370 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 16:04:58 -0400
Subject: [PATCH 110/170] update docstring

---
 deepxde/model.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 19335a9d7..df4fc82e4 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -536,14 +536,14 @@ def train(
             batch_size: Integer, tuple, or ``None``.
 
                 - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`, and instead use
-                `dde.callbacks.PDEResidualResampler
-                <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
-                see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
+                  `dde.callbacks.PDEResidualResampler
+                  <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
+                  see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
 
                 - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
-                then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
-                set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
-                and the second number is the batch size for the trunk net input.
+                  then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
+                  set `batch_size` as a tuple, where the fist number is the batch size for the branch net input
+                  and the second number is the batch size for the trunk net input.
             display_every (Integer): Print the loss and metrics every this steps.
             disregard_previous_best: If ``True``, disregard the previous saved best
                 model.

From 1c17fd90786bc5f64ff1d9e0c08f4aabdc8a9ef5 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 17:01:54 -0400
Subject: [PATCH 111/170] update docstring

---
 deepxde/model.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index df4fc82e4..95d739a43 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -539,7 +539,6 @@ def train(
                   `dde.callbacks.PDEResidualResampler
                   <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                   see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.
-
                 - For DeepONet in the format of Cartesian product, if `batch_size` is an Integer,
                   then it is the batch size for the branch input; if you want to also use mini-batch for the trunk net input,
                   set `batch_size` as a tuple, where the fist number is the batch size for the branch net input

From 0ff3642574eb060a5683775e0fb5a0f29852408d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 17:02:59 -0400
Subject: [PATCH 112/170] update docstring

---
 deepxde/data/quadruple.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py
index a69cac237..2e9fc0ee4 100644
--- a/deepxde/data/quadruple.py
+++ b/deepxde/data/quadruple.py
@@ -48,7 +48,7 @@ class QuadrupleCartesianProd(Data):
     Args:
         X_train: A tuple of three NumPy arrays. The first element has the shape (`N1`,
             `dim1`), the second element has the shape (`N1`, `dim2`), and the third
-            element has the shape (`N2`, `dim3`).The mini-batch is applied to `N1` and `N2`.
+            element has the shape (`N2`, `dim3`).
         y_train: A NumPy array of shape (`N1`, `N2`).
     """
 

From 94ba9e2bb75fc343f8d2d824d1b20c8d18327dd8 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 22 Oct 2022 17:04:40 -0400
Subject: [PATCH 113/170] update docstring

---
 deepxde/model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/model.py b/deepxde/model.py
index 95d739a43..92d16b92f 100755
--- a/deepxde/model.py
+++ b/deepxde/model.py
@@ -535,7 +535,7 @@ def train(
                 of times the network weights are updated.
             batch_size: Integer, tuple, or ``None``.
 
-                - If you solve PDEs via ``dde.data.PDE`` or``dde.data.TimePDE``, do not use `batch_size`, and instead use
+                - If you solve PDEs via ``dde.data.PDE`` or ``dde.data.TimePDE``, do not use `batch_size`, and instead use
                   `dde.callbacks.PDEResidualResampler
                   <https://deepxde.readthedocs.io/en/latest/modules/deepxde.html#deepxde.callbacks.PDEResidualResampler>`_,
                   see an `example <https://github.com/lululxvi/deepxde/blob/master/examples/diffusion_1d_resample.py>`_.

From dcf2a88de117f517003727ba857a6cf94bc3d511 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 10 Nov 2022 12:27:19 -0500
Subject: [PATCH 114/170] fix bug in Pytorch backend DeepONet

---
 deepxde/nn/pytorch/deeponet.py | 4 ++--
 deepxde/nn/pytorch/mionet.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 178b26012..065cffd44 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -42,7 +42,7 @@ def __init__(
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.tensor(0.0, requires_grad=True)
+        self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
 
     def forward(self, inputs):
@@ -120,7 +120,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.tensor(0.0, requires_grad=True)
+            self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
 
     def forward(self, inputs):
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index ff0e8faf3..d75260cb5 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -75,7 +75,7 @@ def __init__(
         else:
             self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.tensor(0.0, requires_grad=True)
+        self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation
@@ -206,7 +206,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.tensor(0.0, requires_grad=True)
+            self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation

From eeb022b9270c8e4632ab74cbcfd6663cca3a7cb8 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 10 Nov 2022 12:37:11 -0500
Subject: [PATCH 115/170] fix bug in Pytorch backend DeepONet

---
 deepxde/nn/pytorch/deeponet.py | 4 ++--
 deepxde/nn/pytorch/mionet.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 065cffd44..50eaf629a 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -42,7 +42,7 @@ def __init__(
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
+        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
 
     def forward(self, inputs):
@@ -120,7 +120,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
+            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
 
     def forward(self, inputs):
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index d75260cb5..db292b6ca 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -75,7 +75,7 @@ def __init__(
         else:
             self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
+        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation
@@ -206,7 +206,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.nn.Parameter(torch.tensor(0.0), requires_grad=True)
+            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation

From 82e58a2f025bfec8f790770e38bcba0546485584 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 19:42:31 -0400
Subject: [PATCH 116/170] Fix merge conflict

---
 deepxde/nn/pytorch/deeponet.py | 4 ++--
 deepxde/nn/pytorch/mionet.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deepxde/nn/pytorch/deeponet.py b/deepxde/nn/pytorch/deeponet.py
index 50eaf629a..7e1a75a66 100755
--- a/deepxde/nn/pytorch/deeponet.py
+++ b/deepxde/nn/pytorch/deeponet.py
@@ -42,7 +42,7 @@ def __init__(
             # Fully connected network
             self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer)
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
+        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0))
         self.regularizer = regularization
 
     def forward(self, inputs):
@@ -120,7 +120,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
+            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0))
         self.regularizer = regularization
 
     def forward(self, inputs):
diff --git a/deepxde/nn/pytorch/mionet.py b/deepxde/nn/pytorch/mionet.py
index db292b6ca..d30f3b663 100644
--- a/deepxde/nn/pytorch/mionet.py
+++ b/deepxde/nn/pytorch/mionet.py
@@ -75,7 +75,7 @@ def __init__(
         else:
             self.output_merger = None
         self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer)
-        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
+        self.b = torch.nn.parameter.Parameter(torch.tensor(0.0))
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation
@@ -206,7 +206,7 @@ def __init__(
             self.trunk = FNN(
                 layer_sizes_trunk, self.activation_trunk, kernel_initializer
             )
-            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0), requires_grad=True)
+            self.b = torch.nn.parameter.Parameter(torch.tensor(0.0))
         self.regularizer = regularization
         self.trunk_last_activation = trunk_last_activation
         self.merge_operation = merge_operation

From d18fe23c022b8b1f28f61a833f9f839b8fd8c8c4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 20:05:40 -0400
Subject: [PATCH 117/170] update research.rst

---
 docs/user/research.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/user/research.rst b/docs/user/research.rst
index fe682bc31..41a575225 100644
--- a/docs/user/research.rst
+++ b/docs/user/research.rst
@@ -200,6 +200,7 @@ PINN
 DeepONet
 --------
 
+#. Z\. Jiang, M. Zhu, D. Li, Q. Li, Y. Yuan, & L. Lu. `Fourier-MIONet: Fourier-enhanced multiple-input neural operators for multiphase modeling of geological carbon sequestration <https://arxiv.org/pdf/2303.04778.pdf>`_. *arXiv preprint arXiv:2303.04778*, 2023.
 #. S\. Wang, & P. Perdikaris. `Long-time integration of parametric evolution equations with physics-informed deeponets <https://www.sciencedirect.com/science/article/pii/S0021999122009184>`_. *Journal of Computational Physics*, 475, p.111855, 2023.
 #. K\. Kobayashi, J. Daniell, & S. Alam. `Operator learning framework for digital twin and complex engineering systems <https://arxiv.org/pdf/2301.06701.pdf>`_. *arXiv e-prints*, pp.arXiv-2301, 2023.
 #. E\. Pickering, S. Guth, G. Karniadakis, & T. Sapsis. `Discovering and forecasting extreme events via active learning in neural operators <https://www.nature.com/articles/s43588-022-00376-0>`_. *Nature Computational Science*, 2(12), pp.823-833, 2022.

From 51b25d32d4826a86dd1eb93b7315607f10c2c231 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 20:10:28 -0400
Subject: [PATCH 118/170] update index.rst

---
 docs/index.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/index.rst b/docs/index.rst
index 3be2aaa4d..6767132e9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -23,6 +23,7 @@ DeepXDE
     - multifidelity DeepONet [`Phys. Rev. Research <https://doi.org/10.1103/PhysRevResearch.4.023210>`_]
     - DeepM&Mnet: solving multiphysics and multiscale problems [`J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2021.110296>`_, `J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2021.110698>`_]
     - Reliable extrapolation [`arXiv <https://arxiv.org/abs/2212.06347>`_]
+    - Fourier-MIONet [`arXiv <https://arxiv.org/abs/2303.04778>`_]
 - multifidelity neural network (MFNN)
     - learning from multifidelity data [`J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2019.109020>`_, `PNAS <https://doi.org/10.1073/pnas.1922210117>`_]
 

From 3297834f56f31163bfbd04cec7a083f7c1afbb9f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 20:14:00 -0400
Subject: [PATCH 119/170] update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 4f63ee93e..edecbcd4f 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,7 @@ DeepXDE is a library for scientific machine learning and physics-informed learni
     - multifidelity DeepONet [[Phys. Rev. Research](https://doi.org/10.1103/PhysRevResearch.4.023210)]
     - DeepM&Mnet: solving multiphysics and multiscale problems [[J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2021.110296), [J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2021.110698)]
     - Reliable extrapolation [[arXiv](https://arxiv.org/abs/2212.06347)]
+    - Fourier-MIONet [[arXiv](https://arxiv.org/abs/2303.04778)]
 - multifidelity neural network (MFNN)
     - learning from multifidelity data [[J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2019.109020), [PNAS](https://doi.org/10.1073/pnas.1922210117)]
 

From 24be4a8926dd10029e21c595145d94f80581926d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 20:42:26 -0400
Subject: [PATCH 120/170] Move Fourier-MIONet after MIONet

---
 README.md      | 2 +-
 docs/index.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index edecbcd4f..6f8dc9890 100644
--- a/README.md
+++ b/README.md
@@ -27,11 +27,11 @@ DeepXDE is a library for scientific machine learning and physics-informed learni
     - DeepONet: learning operators [[Nat. Mach. Intell.](https://doi.org/10.1038/s42256-021-00302-5)]
     - DeepONet extensions, e.g., POD-DeepONet [[Comput. Methods Appl. Mech. Eng.](https://doi.org/10.1016/j.cma.2022.114778)]
     - MIONet: learning multiple-input operators [[SIAM J. Sci. Comput.](https://doi.org/10.1137/22M1477751)]
+    - Fourier-MIONet [[arXiv](https://arxiv.org/abs/2303.04778)]
     - physics-informed DeepONet [[Sci. Adv.](https://doi.org/10.1126/sciadv.abi8605)]
     - multifidelity DeepONet [[Phys. Rev. Research](https://doi.org/10.1103/PhysRevResearch.4.023210)]
     - DeepM&Mnet: solving multiphysics and multiscale problems [[J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2021.110296), [J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2021.110698)]
     - Reliable extrapolation [[arXiv](https://arxiv.org/abs/2212.06347)]
-    - Fourier-MIONet [[arXiv](https://arxiv.org/abs/2303.04778)]
 - multifidelity neural network (MFNN)
     - learning from multifidelity data [[J. Comput. Phys.](https://doi.org/10.1016/j.jcp.2019.109020), [PNAS](https://doi.org/10.1073/pnas.1922210117)]
 
diff --git a/docs/index.rst b/docs/index.rst
index 6767132e9..d90fa7d0c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -19,11 +19,11 @@ DeepXDE
     - DeepONet: learning operators [`Nat. Mach. Intell. <https://doi.org/10.1038/s42256-021-00302-5>`_]
     - DeepONet extensions, e.g., POD-DeepONet [`Comput. Methods Appl. Mech. Eng. <https://doi.org/10.1016/j.cma.2022.114778>`_]
     - MIONet: learning multiple-input operators [`SIAM J. Sci. Comput. <https://doi.org/10.1137/22M1477751>`_]
+    - Fourier-MIONet [`arXiv <https://arxiv.org/abs/2303.04778>`_]
     - physics-informed DeepONet [`Sci. Adv. <https://doi.org/10.1126/sciadv.abi8605>`_]
     - multifidelity DeepONet [`Phys. Rev. Research <https://doi.org/10.1103/PhysRevResearch.4.023210>`_]
     - DeepM&Mnet: solving multiphysics and multiscale problems [`J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2021.110296>`_, `J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2021.110698>`_]
     - Reliable extrapolation [`arXiv <https://arxiv.org/abs/2212.06347>`_]
-    - Fourier-MIONet [`arXiv <https://arxiv.org/abs/2303.04778>`_]
 - multifidelity neural network (MFNN)
     - learning from multifidelity data [`J. Comput. Phys. <https://doi.org/10.1016/j.jcp.2019.109020>`_, `PNAS <https://doi.org/10.1073/pnas.1922210117>`_]
 

From 3980dff6c671967220c355117b212b779e0db731 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 20:58:08 -0400
Subject: [PATCH 121/170] update university and industry list

---
 docs/user/research.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/user/research.rst b/docs/user/research.rst
index 41a575225..952602ebe 100644
--- a/docs/user/research.rst
+++ b/docs/user/research.rst
@@ -42,6 +42,7 @@ DeepXDE has been used in
   `University of Electronic Science and Technology of China <https://en.uestc.edu.cn>`_,
   `KTH Royal Institute of Technology <https://www.kth.se/en>`_,
   `Florida State University <https://www.fsu.edu>`_,
+  `University of Delaware <https://www.udel.edu>`_,
   `Xiamen University <https://en.xmu.edu.cn>`_,
   `Beihang University <https://ev.buaa.edu.cn>`_,
   `University of Strasbourg <https://en.unistra.fr>`_,
@@ -103,7 +104,8 @@ DeepXDE has been used in
   `BirenTech Research <https://www.birentech.com>`_,
   `General Motors <https://www.gm.com>`_,
   `RocketML <https://www.rocketml.net>`_,
-  `Saudi Aramco <https://www.aramco.com>`_
+  `Saudi Aramco <https://www.aramco.com>`_,
+  `ExxonMobil <https://corporate.exxonmobil.com>`_
 
 Here is a list of research papers that used DeepXDE. If you would like your paper to appear here, open an issue in the GitHub "Issues" section.
 

From 466a0763bb3966998d88fd50a0d14dd83eec9231 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 21:16:16 -0400
Subject: [PATCH 122/170] Move ExxonMobil after BirenTech Research

---
 docs/user/research.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/user/research.rst b/docs/user/research.rst
index 952602ebe..ddb91af7f 100644
--- a/docs/user/research.rst
+++ b/docs/user/research.rst
@@ -102,10 +102,10 @@ DeepXDE has been used in
   `Anailytica <https://anailytica.com>`_,
   `Ansys <https://www.ansys.com>`_,
   `BirenTech Research <https://www.birentech.com>`_,
+  `ExxonMobil <https://corporate.exxonmobil.com>`_,
   `General Motors <https://www.gm.com>`_,
   `RocketML <https://www.rocketml.net>`_,
-  `Saudi Aramco <https://www.aramco.com>`_,
-  `ExxonMobil <https://corporate.exxonmobil.com>`_
+  `Saudi Aramco <https://www.aramco.com>`_
 
 Here is a list of research papers that used DeepXDE. If you would like your paper to appear here, open an issue in the GitHub "Issues" section.
 

From 6cadf2fe3bd384fe81919f479e7746e837b8cb3d Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Thu, 13 Apr 2023 21:20:43 -0400
Subject: [PATCH 123/170] update university list

---
 docs/user/research.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/user/research.rst b/docs/user/research.rst
index ddb91af7f..8fc6ff69e 100644
--- a/docs/user/research.rst
+++ b/docs/user/research.rst
@@ -42,7 +42,6 @@ DeepXDE has been used in
   `University of Electronic Science and Technology of China <https://en.uestc.edu.cn>`_,
   `KTH Royal Institute of Technology <https://www.kth.se/en>`_,
   `Florida State University <https://www.fsu.edu>`_,
-  `University of Delaware <https://www.udel.edu>`_,
   `Xiamen University <https://en.xmu.edu.cn>`_,
   `Beihang University <https://ev.buaa.edu.cn>`_,
   `University of Strasbourg <https://en.unistra.fr>`_,
@@ -51,6 +50,7 @@ DeepXDE has been used in
   `University of Rome Tor Vergata <http://web.uniroma2.it>`_,
   `Chalmers University of Technology <https://www.chalmers.se/en>`_,
   `Wuhan University of Technology <http://english.whut.edu.cn>`_,
+  `University of Delaware <https://www.udel.edu>`_,
   `University of Kentucky <https://www.uky.edu>`_,
   `University of Surrey <https://www.surrey.ac.uk>`_,
   `Federal University of Rio de Janeiro <https://ufrj.br/en>`_,

From 8089a23a53bafbf569c16d82685c3594e64feb6e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 1 Dec 2024 15:55:42 -0500
Subject: [PATCH 124/170] update pde_operator.py to support forward mode AD

---
 deepxde/data/pde_operator.py | 98 ++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 32 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index ce573274d..1b9cda494 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -237,49 +237,83 @@ def __init__(
         self.train_next_batch()
         self.test()
 
-    def _losses(self, outputs, loss_fn, inputs, model, num_func):
+    def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
 
-        losses = []
-        for i in range(num_func):
-            out = outputs[i]
-            # Single output
-            if bkd.ndim(out) == 1:
-                out = out[:, None]
-            f = []
+        if config.autodiff == "forward": # forward mode AD
+            losses=[]
+            def forward_call(trunk_input):
+                return aux[0]((inputs[0], trunk_input))
+
             if self.pde.pde is not None:
-                f = self.pde.pde(inputs[1], out, model.net.auxiliary_vars[i][:, None])
+                f = self.pde.pde(inputs[1], (outputs, forward_call), model.net.auxiliary_vars)
                 if not isinstance(f, (list, tuple)):
                     f = [f]
-            error_f = [fi[bcs_start[-1] :] for fi in f]
-            losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
-
-            for j, bc in enumerate(self.pde.bcs):
-                beg, end = bcs_start[j], bcs_start[j + 1]
-                # The same BC points are used for training and testing.
-                error = bc.error(
-                    self.train_x[1],
-                    inputs[1],
-                    out,
-                    beg,
-                    end,
-                    aux_var=model.net.auxiliary_vars[i][:, None],
-                )
-                losses_i.append(loss_fn(bkd.zeros_like(error), error))
-
-            losses.append(losses_i)
-
-        losses = zip(*losses)
-        # Use stack instead of as_tensor to keep the gradients.
-        losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
+            bcs_start = np.cumsum([0] + self.pde.num_bcs)
+            error_f = [fi[:, bcs_start[-1]:] for fi in f]
+            losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
+            # BC
+            for k, bc in enumerate(self.pde.bcs):
+                beg, end = bcs_start[k], bcs_start[k + 1]
+                error_k = []
+                # NOTE: this loop over functions can also be avoided if we implement collective ic/bc
+                for i in range(num_func):
+                    output_i = outputs[i]
+                    if bkd.ndim(output_i) == 1:  # noqa
+                        output_i = output_i[:, None]
+                    error_ki = bc.error(
+                        self.train_x[1],
+                        inputs[1],
+                        output_i,
+                        beg,
+                        end,
+                        aux_var=model.net.auxiliary_vars[i][:, None],
+                    )
+                    error_k.append(error_ki)
+                error_k = bkd.stack(error_k, axis=0)  # noqa
+                loss_k = loss_fn(bkd.zeros_like(error_k), error_k)  # noqa
+                losses.append(loss_k)
+        else: # reverse mode AD
+            losses = []
+            for i in range(num_func):
+                out = outputs[i]
+                # Single output
+                if bkd.ndim(out) == 1:
+                    out = out[:, None]
+                f = []
+                if self.pde.pde is not None:
+                    f = self.pde.pde(inputs[1], out, model.net.auxiliary_vars[i][:, None])
+                    if not isinstance(f, (list, tuple)):
+                        f = [f]
+                error_f = [fi[bcs_start[-1] :] for fi in f]
+                losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
+
+                for j, bc in enumerate(self.pde.bcs):
+                    beg, end = bcs_start[j], bcs_start[j + 1]
+                    # The same BC points are used for training and testing.
+                    error = bc.error(
+                        self.train_x[1],
+                        inputs[1],
+                        out,
+                        beg,
+                        end,
+                        aux_var=model.net.auxiliary_vars[i][:, None],
+                    )
+                    losses_i.append(loss_fn(bkd.zeros_like(error), error))
+
+                losses.append(losses_i)
+
+            losses = zip(*losses)
+            # Use stack instead of as_tensor to keep the gradients.
+            losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):
         num_func = self.num_func if self.batch_size is None else self.batch_size
-        return self._losses(outputs, loss_fn, inputs, model, num_func)
+        return self._losses(outputs, loss_fn, inputs, model, num_func, aux=aux)
 
     def losses_test(self, targets, outputs, loss_fn, inputs, model, aux=None):
-        return self._losses(outputs, loss_fn, inputs, model, len(self.test_x[0]))
+        return self._losses(outputs, loss_fn, inputs, model, len(self.test_x[0]), aux=aux)
 
     def train_next_batch(self, batch_size=None):
         if self.train_x is None:

From 18ff1c106774a376a713babc7b0a32d65671f9ce Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 1 Dec 2024 16:05:38 -0500
Subject: [PATCH 125/170] fix the bug when multi output needed

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index b58aa7621..3beedb3ad 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -94,7 +94,7 @@ def grad_fn(x):
                 # which is also a tuple. This is useful for further computation, e.g.,
                 # Hessian.
                 self.J[i, j] = (
-                    self.J[j][0][:, i : i + 1],
+                    self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],
                 )
         return self.J[i, j]

From 6c258db879cce0c3ad9be1859db45d6ea6c5ad22 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 1 Dec 2024 16:27:55 -0500
Subject: [PATCH 126/170] add a blank line to fix Codacy Static Code Analysis

---
 deepxde/data/pde_operator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 1b9cda494..50a97b27f 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -242,6 +242,7 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
 
         if config.autodiff == "forward": # forward mode AD
             losses=[]
+
             def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 

From 5917be1e0e4b16285f76edb6ba1c2b99c2555fbf Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 1 Dec 2024 21:13:51 -0500
Subject: [PATCH 127/170] Use reverse mode in the if, forward mode in the else.

---
 deepxde/data/pde_operator.py | 68 ++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 50a97b27f..c5d216f22 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -240,7 +240,40 @@ def __init__(
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
 
-        if config.autodiff == "forward": # forward mode AD
+        if config.autodiff == "reverse": # reverse mode AD
+            losses = []
+            for i in range(num_func):
+                out = outputs[i]
+                # Single output
+                if bkd.ndim(out) == 1:
+                    out = out[:, None]
+                f = []
+                if self.pde.pde is not None:
+                    f = self.pde.pde(inputs[1], out, model.net.auxiliary_vars[i][:, None])
+                    if not isinstance(f, (list, tuple)):
+                        f = [f]
+                error_f = [fi[bcs_start[-1]:] for fi in f]
+                losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
+
+                for j, bc in enumerate(self.pde.bcs):
+                    beg, end = bcs_start[j], bcs_start[j + 1]
+                    # The same BC points are used for training and testing.
+                    error = bc.error(
+                        self.train_x[1],
+                        inputs[1],
+                        out,
+                        beg,
+                        end,
+                        aux_var=model.net.auxiliary_vars[i][:, None],
+                    )
+                    losses_i.append(loss_fn(bkd.zeros_like(error), error))
+
+                losses.append(losses_i)
+
+            losses = zip(*losses)
+            # Use stack instead of as_tensor to keep the gradients.
+            losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
+        else: # forward mode AD
             losses=[]
 
             def forward_call(trunk_input):
@@ -274,39 +307,6 @@ def forward_call(trunk_input):
                 error_k = bkd.stack(error_k, axis=0)  # noqa
                 loss_k = loss_fn(bkd.zeros_like(error_k), error_k)  # noqa
                 losses.append(loss_k)
-        else: # reverse mode AD
-            losses = []
-            for i in range(num_func):
-                out = outputs[i]
-                # Single output
-                if bkd.ndim(out) == 1:
-                    out = out[:, None]
-                f = []
-                if self.pde.pde is not None:
-                    f = self.pde.pde(inputs[1], out, model.net.auxiliary_vars[i][:, None])
-                    if not isinstance(f, (list, tuple)):
-                        f = [f]
-                error_f = [fi[bcs_start[-1] :] for fi in f]
-                losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
-
-                for j, bc in enumerate(self.pde.bcs):
-                    beg, end = bcs_start[j], bcs_start[j + 1]
-                    # The same BC points are used for training and testing.
-                    error = bc.error(
-                        self.train_x[1],
-                        inputs[1],
-                        out,
-                        beg,
-                        end,
-                        aux_var=model.net.auxiliary_vars[i][:, None],
-                    )
-                    losses_i.append(loss_fn(bkd.zeros_like(error), error))
-
-                losses.append(losses_i)
-
-            losses = zip(*losses)
-            # Use stack instead of as_tensor to keep the gradients.
-            losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From 263202644734d71f9d7ac2d23a5d788f0fec889f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 8 Dec 2024 18:32:06 -0500
Subject: [PATCH 128/170] rollback

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index 3beedb3ad..b58aa7621 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -94,7 +94,7 @@ def grad_fn(x):
                 # which is also a tuple. This is useful for further computation, e.g.,
                 # Hessian.
                 self.J[i, j] = (
-                    self.J[j][0][..., i : i + 1],
+                    self.J[j][0][:, i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],
                 )
         return self.J[i, j]

From 07a9d96483da6beccde749026947d5293fea0954 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 8 Dec 2024 18:43:11 -0500
Subject: [PATCH 129/170] add elif config.autodiff == "forward":

---
 deepxde/data/pde_operator.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index c5d216f22..429aa1a52 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -273,7 +273,7 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             losses = zip(*losses)
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
-        else: # forward mode AD
+        elif config.autodiff == "forward": # forward mode AD
             losses=[]
 
             def forward_call(trunk_input):
@@ -290,7 +290,6 @@ def forward_call(trunk_input):
             for k, bc in enumerate(self.pde.bcs):
                 beg, end = bcs_start[k], bcs_start[k + 1]
                 error_k = []
-                # NOTE: this loop over functions can also be avoided if we implement collective ic/bc
                 for i in range(num_func):
                     output_i = outputs[i]
                     if bkd.ndim(output_i) == 1:  # noqa

From faca8e7a5d80b4c733334436783505775c379d4f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 10 Dec 2024 20:33:03 -0500
Subject: [PATCH 130/170] add losses = [] before if-else

---
 deepxde/data/pde_operator.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 429aa1a52..5eb7efe0c 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -239,9 +239,9 @@ def __init__(
 
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
+        losses = []
 
         if config.autodiff == "reverse": # reverse mode AD
-            losses = []
             for i in range(num_func):
                 out = outputs[i]
                 # Single output
@@ -274,7 +274,6 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward": # forward mode AD
-            losses=[]
 
             def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))

From 45fecb58c81fc612abd1b42dfb2e8df2326d1274 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 15:47:00 -0500
Subject: [PATCH 131/170] delete empty line

---
 deepxde/data/pde_operator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 5eb7efe0c..035091817 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -240,7 +240,6 @@ def __init__(
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
         losses = []
-
         if config.autodiff == "reverse": # reverse mode AD
             for i in range(num_func):
                 out = outputs[i]

From cf076d9907b7d2a217a24fc23fababcb31ee724c Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 16:12:53 -0500
Subject: [PATCH 132/170] delete duplicated code

---
 deepxde/data/pde_operator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 035091817..d3f6cc1ec 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -281,7 +281,6 @@ def forward_call(trunk_input):
                 f = self.pde.pde(inputs[1], (outputs, forward_call), model.net.auxiliary_vars)
                 if not isinstance(f, (list, tuple)):
                     f = [f]
-            bcs_start = np.cumsum([0] + self.pde.num_bcs)
             error_f = [fi[:, bcs_start[-1]:] for fi in f]
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
             # BC

From 947abee91afa5612a44683891896ed1ed078daf8 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 19:01:37 -0500
Subject: [PATCH 133/170] Refactor the code to reduce the redundancy

---
 deepxde/data/pde_operator.py | 55 ++++++++++++++----------------------
 1 file changed, 21 insertions(+), 34 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index d3f6cc1ec..d40bdbf4b 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -240,6 +240,7 @@ def __init__(
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
         losses = []
+        # PDE loss
         if config.autodiff == "reverse": # reverse mode AD
             for i in range(num_func):
                 out = outputs[i]
@@ -253,20 +254,6 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
                         f = [f]
                 error_f = [fi[bcs_start[-1]:] for fi in f]
                 losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
-
-                for j, bc in enumerate(self.pde.bcs):
-                    beg, end = bcs_start[j], bcs_start[j + 1]
-                    # The same BC points are used for training and testing.
-                    error = bc.error(
-                        self.train_x[1],
-                        inputs[1],
-                        out,
-                        beg,
-                        end,
-                        aux_var=model.net.auxiliary_vars[i][:, None],
-                    )
-                    losses_i.append(loss_fn(bkd.zeros_like(error), error))
-
                 losses.append(losses_i)
 
             losses = zip(*losses)
@@ -283,26 +270,26 @@ def forward_call(trunk_input):
                     f = [f]
             error_f = [fi[:, bcs_start[-1]:] for fi in f]
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
-            # BC
-            for k, bc in enumerate(self.pde.bcs):
-                beg, end = bcs_start[k], bcs_start[k + 1]
-                error_k = []
-                for i in range(num_func):
-                    output_i = outputs[i]
-                    if bkd.ndim(output_i) == 1:  # noqa
-                        output_i = output_i[:, None]
-                    error_ki = bc.error(
-                        self.train_x[1],
-                        inputs[1],
-                        output_i,
-                        beg,
-                        end,
-                        aux_var=model.net.auxiliary_vars[i][:, None],
-                    )
-                    error_k.append(error_ki)
-                error_k = bkd.stack(error_k, axis=0)  # noqa
-                loss_k = loss_fn(bkd.zeros_like(error_k), error_k)  # noqa
-                losses.append(loss_k)
+        # BC loss
+        for k, bc in enumerate(self.pde.bcs):
+            beg, end = bcs_start[k], bcs_start[k + 1]
+            error_k = []
+            for i in range(num_func):
+                output_i = outputs[i]
+                if bkd.ndim(output_i) == 1:  # noqa
+                    output_i = output_i[:, None]
+                error_ki = bc.error(
+                    self.train_x[1],
+                    inputs[1],
+                    output_i,
+                    beg,
+                    end,
+                    aux_var=model.net.auxiliary_vars[i][:, None],
+                )
+                error_k.append(error_ki)
+            error_k = bkd.stack(error_k, axis=0)  # noqa
+            loss_k = loss_fn(bkd.zeros_like(error_k), error_k)  # noqa
+            losses.append(loss_k)
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From cead3845737f17808c324e2af2b8059b1e1a6761 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 19:21:49 -0500
Subject: [PATCH 134/170] add empty line

---
 deepxde/data/pde_operator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index d40bdbf4b..9523d00ea 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -270,6 +270,7 @@ def forward_call(trunk_input):
                     f = [f]
             error_f = [fi[:, bcs_start[-1]:] for fi in f]
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
+
         # BC loss
         for k, bc in enumerate(self.pde.bcs):
             beg, end = bcs_start[k], bcs_start[k + 1]

From 68b4fe9816ab3fce198fd9118c585c5347568c5f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 19:30:45 -0500
Subject: [PATCH 135/170] add empty line and change variable names

---
 deepxde/data/pde_operator.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 9523d00ea..5a27f802a 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -272,25 +272,25 @@ def forward_call(trunk_input):
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
 
         # BC loss
-        for k, bc in enumerate(self.pde.bcs):
-            beg, end = bcs_start[k], bcs_start[k + 1]
-            error_k = []
+        for j, bc in enumerate(self.pde.bcs):
+            beg, end = bcs_start[j], bcs_start[j + 1]
+            error = []
             for i in range(num_func):
-                output_i = outputs[i]
-                if bkd.ndim(output_i) == 1:  # noqa
-                    output_i = output_i[:, None]
-                error_ki = bc.error(
+                out = outputs[i]
+                if bkd.ndim(out) == 1:  # noqa
+                    out = out[:, None]
+                error_i = bc.error(
                     self.train_x[1],
                     inputs[1],
-                    output_i,
+                    out,
                     beg,
                     end,
                     aux_var=model.net.auxiliary_vars[i][:, None],
                 )
-                error_k.append(error_ki)
-            error_k = bkd.stack(error_k, axis=0)  # noqa
-            loss_k = loss_fn(bkd.zeros_like(error_k), error_k)  # noqa
-            losses.append(loss_k)
+                error.append(error_i)
+            error = bkd.stack(error, axis=0)  # noqa
+            loss = loss_fn(bkd.zeros_like(error), error)  # noqa
+            losses.append(loss)
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From c1b3384e9b33e0673380f09cac6b4597fb1c3443 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 23:00:09 -0500
Subject: [PATCH 136/170] polish code

---
 deepxde/data/pde_operator.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 5a27f802a..2bc9d57a9 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -5,6 +5,7 @@
 from .. import backend as bkd
 from .. import config
 from ..utils import run_if_all_none
+import time
 
 
 class PDEOperator(Data):
@@ -237,6 +238,7 @@ def __init__(
         self.train_next_batch()
         self.test()
 
+
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
         losses = []
@@ -269,7 +271,7 @@ def forward_call(trunk_input):
                 if not isinstance(f, (list, tuple)):
                     f = [f]
             error_f = [fi[:, bcs_start[-1]:] for fi in f]
-            losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]  # noqa
+            losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
 
         # BC loss
         for j, bc in enumerate(self.pde.bcs):
@@ -277,7 +279,7 @@ def forward_call(trunk_input):
             error = []
             for i in range(num_func):
                 out = outputs[i]
-                if bkd.ndim(out) == 1:  # noqa
+                if bkd.ndim(out) == 1:
                     out = out[:, None]
                 error_i = bc.error(
                     self.train_x[1],
@@ -288,8 +290,8 @@ def forward_call(trunk_input):
                     aux_var=model.net.auxiliary_vars[i][:, None],
                 )
                 error.append(error_i)
-            error = bkd.stack(error, axis=0)  # noqa
-            loss = loss_fn(bkd.zeros_like(error), error)  # noqa
+            error = bkd.stack(error, 0)
+            loss = loss_fn(bkd.zeros_like(error), error)
             losses.append(loss)
         return losses
 

From 303066a161b65123c23afddc4353126dd4b19761 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 15 Dec 2024 23:18:50 -0500
Subject: [PATCH 137/170] fix bug

---
 deepxde/data/pde_operator.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 2bc9d57a9..4eb0b23d4 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -5,7 +5,6 @@
 from .. import backend as bkd
 from .. import config
 from ..utils import run_if_all_none
-import time
 
 
 class PDEOperator(Data):
@@ -238,7 +237,6 @@ def __init__(
         self.train_next_batch()
         self.test()
 
-
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
         losses = []

From 4d4106c06933ca16abe79355b3ee0c2c5cf322a6 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 16 Dec 2024 15:58:19 -0500
Subject: [PATCH 138/170] Reformat by using black

---
 deepxde/data/pde_operator.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 4eb0b23d4..174df29eb 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -241,7 +241,7 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
         losses = []
         # PDE loss
-        if config.autodiff == "reverse": # reverse mode AD
+        if config.autodiff == "reverse":  # reverse mode AD
             for i in range(num_func):
                 out = outputs[i]
                 # Single output
@@ -249,26 +249,30 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
                     out = out[:, None]
                 f = []
                 if self.pde.pde is not None:
-                    f = self.pde.pde(inputs[1], out, model.net.auxiliary_vars[i][:, None])
+                    f = self.pde.pde(
+                        inputs[1], out, model.net.auxiliary_vars[i][:, None]
+                    )
                     if not isinstance(f, (list, tuple)):
                         f = [f]
-                error_f = [fi[bcs_start[-1]:] for fi in f]
+                error_f = [fi[bcs_start[-1] :] for fi in f]
                 losses_i = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
                 losses.append(losses_i)
 
             losses = zip(*losses)
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
-        elif config.autodiff == "forward": # forward mode AD
+        elif config.autodiff == "forward":  # forward mode AD
 
             def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
             if self.pde.pde is not None:
-                f = self.pde.pde(inputs[1], (outputs, forward_call), model.net.auxiliary_vars)
+                f = self.pde.pde(
+                    inputs[1], (outputs, forward_call), model.net.auxiliary_vars
+                )
                 if not isinstance(f, (list, tuple)):
                     f = [f]
-            error_f = [fi[:, bcs_start[-1]:] for fi in f]
+            error_f = [fi[:, bcs_start[-1] :] for fi in f]
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
 
         # BC loss
@@ -298,7 +302,9 @@ def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):
         return self._losses(outputs, loss_fn, inputs, model, num_func, aux=aux)
 
     def losses_test(self, targets, outputs, loss_fn, inputs, model, aux=None):
-        return self._losses(outputs, loss_fn, inputs, model, len(self.test_x[0]), aux=aux)
+        return self._losses(
+            outputs, loss_fn, inputs, model, len(self.test_x[0]), aux=aux
+        )
 
     def train_next_batch(self, batch_size=None):
         if self.train_x is None:

From e7f2bb16eb8fcce07331a03c6a1e5fbf38e76574 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 16 Dec 2024 22:42:02 -0500
Subject: [PATCH 139/170] add comment

---
 deepxde/data/pde_operator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 174df29eb..5956128a8 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -267,6 +267,7 @@ def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
             if self.pde.pde is not None:
+                # f has the shape (`N1`, `N2`), inputs[1] has the shape (`N2`, `dim2`), outputs has the shape (`N1`, `N2`)
                 f = self.pde.pde(
                     inputs[1], (outputs, forward_call), model.net.auxiliary_vars
                 )

From e490963513653dfb0fb67d21ad438420a25912dd Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 17 Dec 2024 18:01:21 -0500
Subject: [PATCH 140/170] add comment

---
 deepxde/data/pde_operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 5956128a8..f53e541e8 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -267,7 +267,7 @@ def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
             if self.pde.pde is not None:
-                # f has the shape (`N1`, `N2`), inputs[1] has the shape (`N2`, `dim2`), outputs has the shape (`N1`, `N2`)
+                # Each f has the shape (`N1`, `N2`), inputs[1] has the shape (`N2`, `dim2`), outputs has the shape (`N1`, `N2`)
                 f = self.pde.pde(
                     inputs[1], (outputs, forward_call), model.net.auxiliary_vars
                 )

From 106f13febf8d9261e8b0587dc0ac1fbbc510b0cd Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 17 Dec 2024 19:18:04 -0500
Subject: [PATCH 141/170] delete comment

---
 deepxde/data/pde_operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index f53e541e8..cca1b15a0 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -267,7 +267,7 @@ def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
             if self.pde.pde is not None:
-                # Each f has the shape (`N1`, `N2`), inputs[1] has the shape (`N2`, `dim2`), outputs has the shape (`N1`, `N2`)
+                # Each f has the shape (`N1`, `N2`)
                 f = self.pde.pde(
                     inputs[1], (outputs, forward_call), model.net.auxiliary_vars
                 )

From bcac97f260c61bc2afd654be3bf755e969cf7060 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 11:30:09 -0500
Subject: [PATCH 142/170] modify comment

---
 deepxde/data/pde_operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index cca1b15a0..8d0bf97c1 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -267,7 +267,7 @@ def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
             if self.pde.pde is not None:
-                # Each f has the shape (`N1`, `N2`)
+                # Each f has the shape (N1, N2)
                 f = self.pde.pde(
                     inputs[1], (outputs, forward_call), model.net.auxiliary_vars
                 )

From 420313bbd6a3d02f341c281eb995107929514755 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 11:35:49 -0500
Subject: [PATCH 143/170] add comment

---
 deepxde/data/pde_operator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 8d0bf97c1..669cd8a81 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -266,6 +266,7 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             def forward_call(trunk_input):
                 return aux[0]((inputs[0], trunk_input))
 
+            f = []
             if self.pde.pde is not None:
                 # Each f has the shape (N1, N2)
                 f = self.pde.pde(

From 292d05e198e342681f2ee07832242d193e541fa9 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 11:48:38 -0500
Subject: [PATCH 144/170] add comment

---
 deepxde/data/pde_operator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 669cd8a81..a9f7d66e9 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -275,6 +275,7 @@ def forward_call(trunk_input):
                 if not isinstance(f, (list, tuple)):
                     f = [f]
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
+            # Each error has the shape (N1, ~N2)
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
 
         # BC loss

From 85c8d2e0a23f0b7f927b6e0a89b138f60ee04af8 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 12:42:26 -0500
Subject: [PATCH 145/170] refactor code of BC loss

---
 deepxde/data/pde_operator.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index a9f7d66e9..8fb7f1485 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -294,10 +294,8 @@ def forward_call(trunk_input):
                     end,
                     aux_var=model.net.auxiliary_vars[i][:, None],
                 )
-                error.append(error_i)
-            error = bkd.stack(error, 0)
-            loss = loss_fn(bkd.zeros_like(error), error)
-            losses.append(loss)
+                error.append(loss_fn(bkd.zeros_like(error_i), error_i))
+            losses.append(bkd.reduce_mean(bkd.stack(error, 0)))
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From 5ef4f4fb46946cbe01f4b06749e3d6591b24aae4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 16:27:33 -0500
Subject: [PATCH 146/170] modify bc loss code

---
 deepxde/data/pde_operator.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 8fb7f1485..19046904f 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -277,16 +277,16 @@ def forward_call(trunk_input):
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
             # Each error has the shape (N1, ~N2)
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
-
         # BC loss
-        for j, bc in enumerate(self.pde.bcs):
-            beg, end = bcs_start[j], bcs_start[j + 1]
-            error = []
-            for i in range(num_func):
-                out = outputs[i]
-                if bkd.ndim(out) == 1:
-                    out = out[:, None]
-                error_i = bc.error(
+        error_bc = []
+        for i in range(num_func):
+            error_i = []
+            out = outputs[i]
+            if bkd.ndim(out) == 1:
+                out = out[:, None]
+            for j, bc in enumerate(self.pde.bcs):
+                beg, end = bcs_start[j], bcs_start[j + 1]
+                error = bc.error(
                     self.train_x[1],
                     inputs[1],
                     out,
@@ -294,8 +294,13 @@ def forward_call(trunk_input):
                     end,
                     aux_var=model.net.auxiliary_vars[i][:, None],
                 )
-                error.append(loss_fn(bkd.zeros_like(error_i), error_i))
-            losses.append(bkd.reduce_mean(bkd.stack(error, 0)))
+                error_i.append(loss_fn(bkd.zeros_like(error), error))
+
+            error_bc.append(error_i)
+
+        error_bc = zip(*error_bc)
+        error_bc = [bkd.reduce_mean(bkd.stack(error, 0)) for error in error_bc]
+        losses.append(error_bc)
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From 66682a457ca0bcef04e5acd93c058db6cd43ad6a Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 18 Dec 2024 16:39:06 -0500
Subject: [PATCH 147/170] modify code

---
 deepxde/data/pde_operator.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 19046904f..308b8f088 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -239,6 +239,7 @@ def __init__(
 
     def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
         bcs_start = np.cumsum([0] + self.pde.num_bcs)
+
         losses = []
         # PDE loss
         if config.autodiff == "reverse":  # reverse mode AD
@@ -277,10 +278,11 @@ def forward_call(trunk_input):
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
             # Each error has the shape (N1, ~N2)
             losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
+
         # BC loss
         error_bc = []
         for i in range(num_func):
-            error_i = []
+            losses_i = []
             out = outputs[i]
             if bkd.ndim(out) == 1:
                 out = out[:, None]
@@ -294,9 +296,9 @@ def forward_call(trunk_input):
                     end,
                     aux_var=model.net.auxiliary_vars[i][:, None],
                 )
-                error_i.append(loss_fn(bkd.zeros_like(error), error))
+                losses_i.append(loss_fn(bkd.zeros_like(error), error))
 
-            error_bc.append(error_i)
+            error_bc.append(losses_i)
 
         error_bc = zip(*error_bc)
         error_bc = [bkd.reduce_mean(bkd.stack(error, 0)) for error in error_bc]

From a9a9932e868e064912a1306c3a11503187cebac4 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 22 Dec 2024 20:34:32 -0500
Subject: [PATCH 148/170] modify code

---
 deepxde/data/pde_operator.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 308b8f088..1bb97181f 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -275,9 +275,13 @@ def forward_call(trunk_input):
                 )
                 if not isinstance(f, (list, tuple)):
                     f = [f]
-            error_f = [fi[:, bcs_start[-1] :] for fi in f]
+            error_f = [fi[:, bcs_start[-1]:] for fi in f]
             # Each error has the shape (N1, ~N2)
-            losses = [loss_fn(bkd.zeros_like(error), error) for error in error_f]
+            for error in error_f:
+                error_i = []
+                for i in range(error.shape[0]):
+                    error_i.append(loss_fn(bkd.zeros_like(error[i]), error[i]))
+                losses.append(bkd.reduce_mean(bkd.stack(error_i, 0)))
 
         # BC loss
         error_bc = []

From fa6e3681e61094e30a9311dea266b4f614d12261 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 17:40:22 -0500
Subject: [PATCH 149/170] add comment

---
 deepxde/data/pde_operator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 1bb97181f..db504bf6c 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -279,7 +279,7 @@ def forward_call(trunk_input):
             # Each error has the shape (N1, ~N2)
             for error in error_f:
                 error_i = []
-                for i in range(error.shape[0]):
+                for i in range(num_func):
                     error_i.append(loss_fn(bkd.zeros_like(error[i]), error[i]))
                 losses.append(bkd.reduce_mean(bkd.stack(error_i, 0)))
 
@@ -292,6 +292,7 @@ def forward_call(trunk_input):
                 out = out[:, None]
             for j, bc in enumerate(self.pde.bcs):
                 beg, end = bcs_start[j], bcs_start[j + 1]
+                # The same BC points are used for training and testing.
                 error = bc.error(
                     self.train_x[1],
                     inputs[1],

From 37f19a4dbd9b70515501b643d784a252deebaa40 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 17:42:12 -0500
Subject: [PATCH 150/170] modify comment

---
 deepxde/data/pde_operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index db504bf6c..851101b80 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -275,8 +275,8 @@ def forward_call(trunk_input):
                 )
                 if not isinstance(f, (list, tuple)):
                     f = [f]
-            error_f = [fi[:, bcs_start[-1]:] for fi in f]
             # Each error has the shape (N1, ~N2)
+            error_f = [fi[:, bcs_start[-1]:] for fi in f]
             for error in error_f:
                 error_i = []
                 for i in range(num_func):

From 28716c654b874e54afa332fe878f9630d28e7e18 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 17:44:02 -0500
Subject: [PATCH 151/170] after black

---
 deepxde/data/pde_operator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 851101b80..867af5e51 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -276,7 +276,7 @@ def forward_call(trunk_input):
                 if not isinstance(f, (list, tuple)):
                     f = [f]
             # Each error has the shape (N1, ~N2)
-            error_f = [fi[:, bcs_start[-1]:] for fi in f]
+            error_f = [fi[:, bcs_start[-1] :] for fi in f]
             for error in error_f:
                 error_i = []
                 for i in range(num_func):

From bd88ec4a9762a1cb2a749288e953aa51a108a2ad Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 18:44:58 -0500
Subject: [PATCH 152/170] change variable name

---
 deepxde/data/pde_operator.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 867af5e51..3b001cf9a 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -284,7 +284,7 @@ def forward_call(trunk_input):
                 losses.append(bkd.reduce_mean(bkd.stack(error_i, 0)))
 
         # BC loss
-        error_bc = []
+        losses_bc = []
         for i in range(num_func):
             losses_i = []
             out = outputs[i]
@@ -303,11 +303,11 @@ def forward_call(trunk_input):
                 )
                 losses_i.append(loss_fn(bkd.zeros_like(error), error))
 
-            error_bc.append(losses_i)
+            losses_bc.append(losses_i)
 
-        error_bc = zip(*error_bc)
-        error_bc = [bkd.reduce_mean(bkd.stack(error, 0)) for error in error_bc]
-        losses.append(error_bc)
+        losses_bc = zip(*losses_bc)
+        losses_bc = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses_bc]
+        losses.append(losses_bc)
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From 07f15a5314846facbfc135ab40337f39c7c5fb4b Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 19:31:59 -0500
Subject: [PATCH 153/170] fix a bug of forward-mode AD when muti-output is
 needed

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index b58aa7621..3beedb3ad 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -94,7 +94,7 @@ def grad_fn(x):
                 # which is also a tuple. This is useful for further computation, e.g.,
                 # Hessian.
                 self.J[i, j] = (
-                    self.J[j][0][:, i : i + 1],
+                    self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],
                 )
         return self.J[i, j]

From d3579dd333b158536c94ef6bd666e1409c599333 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 20:00:12 -0500
Subject: [PATCH 154/170] add comment

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index 3beedb3ad..75f50ebf3 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -92,7 +92,7 @@ def grad_fn(x):
                 # In backend tensorflow/pytorch/jax, a tuple of a tensor/tensor/array
                 # and a callable is returned, so that it is consistent with the argument,
                 # which is also a tuple. This is useful for further computation, e.g.,
-                # Hessian.
+                # Hessian. The code still works even the output dim is > 2
                 self.J[i, j] = (
                     self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],

From 00a7be574e9c3df6a315ee326fb4da7b872d0004 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Mon, 23 Dec 2024 20:01:00 -0500
Subject: [PATCH 155/170] add comment

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index 75f50ebf3..8aaf3e3c2 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -92,7 +92,7 @@ def grad_fn(x):
                 # In backend tensorflow/pytorch/jax, a tuple of a tensor/tensor/array
                 # and a callable is returned, so that it is consistent with the argument,
                 # which is also a tuple. This is useful for further computation, e.g.,
-                # Hessian. The code still works even the output dim is > 2
+                # Hessian. The code still works even the output dim is > 2.
                 self.J[i, j] = (
                     self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],

From 2f4e340d2903d5518d90d8a3db8e8cdf88613707 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 24 Dec 2024 23:43:55 -0500
Subject: [PATCH 156/170] update comment

---
 deepxde/gradients/gradients_forward.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index 8aaf3e3c2..2f9aa10f7 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -92,7 +92,9 @@ def grad_fn(x):
                 # In backend tensorflow/pytorch/jax, a tuple of a tensor/tensor/array
                 # and a callable is returned, so that it is consistent with the argument,
                 # which is also a tuple. This is useful for further computation, e.g.,
-                # Hessian. The code still works even the output dim is > 2.
+                # Hessian. The code is designed for the output shape of (batch size, dim),
+                # but we find that the code also works for the output shape of (batch size 1,
+                # batch size 2, dim) such as multiple-output DeepONet.
                 self.J[i, j] = (
                     self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],

From e20364cc69894cfd56e91a0c8a9c53390914e29e Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Wed, 25 Dec 2024 21:57:33 -0500
Subject: [PATCH 157/170] update comment, fix a bug and rewrite the computation
 of dim

---
 deepxde/data/pde_operator.py           | 4 ++--
 deepxde/gradients/gradients_forward.py | 7 ++++---
 deepxde/gradients/jacobian.py          | 7 +++++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 3b001cf9a..9bc5e98d4 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -306,8 +306,8 @@ def forward_call(trunk_input):
             losses_bc.append(losses_i)
 
         losses_bc = zip(*losses_bc)
-        losses_bc = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses_bc]
-        losses.append(losses_bc)
+        for loss in losses_bc:
+            losses.append(bkd.reduce_mean(bkd.stack(loss, 0)))
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):
diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index 2f9aa10f7..cb2b9d008 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -92,9 +92,10 @@ def grad_fn(x):
                 # In backend tensorflow/pytorch/jax, a tuple of a tensor/tensor/array
                 # and a callable is returned, so that it is consistent with the argument,
                 # which is also a tuple. This is useful for further computation, e.g.,
-                # Hessian. The code is designed for the output shape of (batch size, dim),
-                # but we find that the code also works for the output shape of (batch size 1,
-                # batch size 2, dim) such as multiple-output DeepONet.
+                # Hessian. The original code self.J[j][0][:, i : i + 1] is designed for
+                # the output shape of (batch size, dim), but we find that the code
+                # also works for the output shape of (batch size 1, batch size 2, dim)
+                # such as multiple-output DeepONet.
                 self.J[i, j] = (
                     self.J[j][0][..., i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],
diff --git a/deepxde/gradients/jacobian.py b/deepxde/gradients/jacobian.py
index 5b0af016d..7c52f9b6a 100644
--- a/deepxde/gradients/jacobian.py
+++ b/deepxde/gradients/jacobian.py
@@ -2,7 +2,7 @@
 
 from .. import config
 from ..backend import backend_name
-
+from .. import backend as bkd
 
 class Jacobian(ABC):
     """Compute `Jacobian matrix <https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant>`_
@@ -28,7 +28,10 @@ def __init__(self, ys, xs):
             elif config.autodiff == "forward":
                 # For forward-mode AD, a tuple of a tensor and a callable is passed, 
                 # similar to backend jax.
-                self.dim_y = ys[0].shape[1]
+                if bkd.ndim(ys[0]) == 2:
+                    self.dim_y = 1
+                elif bkd.ndim(ys[0]) == 3:
+                    self.dim_y = ys[0].shape[2]
         elif backend_name == "jax":
             # For backend jax, a tuple of a jax array and a callable is passed as one of
             # the arguments, since jax does not support computational graph explicitly.

From 101bc3109fd48243a9831927cabb00e46afcef32 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sat, 28 Dec 2024 16:43:23 -0500
Subject: [PATCH 158/170] update code

---
 deepxde/data/pde_operator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 9bc5e98d4..85174215b 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -306,8 +306,8 @@ def forward_call(trunk_input):
             losses_bc.append(losses_i)
 
         losses_bc = zip(*losses_bc)
-        for loss in losses_bc:
-            losses.append(bkd.reduce_mean(bkd.stack(loss, 0)))
+        losses_bc = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses_bc]
+        losses.extend(losses_bc)
         return losses
 
     def losses_train(self, targets, outputs, loss_fn, inputs, model, aux=None):

From af993772d5e09d6458db72efe135a77f1aa7f587 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 29 Dec 2024 18:20:49 -0500
Subject: [PATCH 159/170] refactor the code for computation of dim_y

---
 deepxde/data/pde_operator.py  | 36 +++++++++++++++++++++++++++++++----
 deepxde/gradients/jacobian.py |  5 +----
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 85174215b..fb47dd803 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -263,18 +263,46 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward":  # forward mode AD
+            if bkd.ndim(outputs) == 2:
+                is_multi_outputs = False
+                shape0, shape1 = outputs.shape[0], outputs.shape[1]
+            elif bkd.ndim(outputs) == 3:
+                is_multi_outputs = True
+                shape0, shape1, shape2 = (
+                    outputs.shape[0],
+                    outputs.shape[1],
+                    outputs.shape[2],
+                )
 
             def forward_call(trunk_input):
-                return aux[0]((inputs[0], trunk_input))
+                output = aux[0]((inputs[0], trunk_input))
+                if not is_multi_outputs:
+                    return output.reshape(shape0 * shape1, 1)
+                elif is_multi_outputs:
+                    return output.reshape(shape0 * shape1, shape2)
+
+            if not is_multi_outputs:
+                outputs = outputs.reshape(shape0 * shape1, 1)
+                auxiliary_vars = model.net.auxiliary_vars.reshape(shape0 * shape1, 1)
+            elif is_multi_outputs:
+                outputs = outputs.reshape(shape0 * shape1, shape2)
+                auxiliary_vars = model.net.auxiliary_vars.reshape(
+                    shape0 * shape1, shape2
+                )
 
             f = []
             if self.pde.pde is not None:
                 # Each f has the shape (N1, N2)
-                f = self.pde.pde(
-                    inputs[1], (outputs, forward_call), model.net.auxiliary_vars
-                )
+                f = self.pde.pde(inputs[1], (outputs, forward_call), auxiliary_vars)
                 if not isinstance(f, (list, tuple)):
                     f = [f]
+
+            if not is_multi_outputs:
+                outputs = outputs.reshape(shape0, shape1)
+                f = [fi.reshape(shape0, shape1) for fi in f]
+            elif is_multi_outputs:
+                outputs = outputs.reshape(shape0, shape1, shape2)
+                f = [fi.reshape(shape0, shape1, shape2) for fi in f]
             # Each error has the shape (N1, ~N2)
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
             for error in error_f:
diff --git a/deepxde/gradients/jacobian.py b/deepxde/gradients/jacobian.py
index 7c52f9b6a..d9e00fd55 100644
--- a/deepxde/gradients/jacobian.py
+++ b/deepxde/gradients/jacobian.py
@@ -28,10 +28,7 @@ def __init__(self, ys, xs):
             elif config.autodiff == "forward":
                 # For forward-mode AD, a tuple of a tensor and a callable is passed, 
                 # similar to backend jax.
-                if bkd.ndim(ys[0]) == 2:
-                    self.dim_y = 1
-                elif bkd.ndim(ys[0]) == 3:
-                    self.dim_y = ys[0].shape[2]
+                self.dim_y = ys[0].shape[-1]
         elif backend_name == "jax":
             # For backend jax, a tuple of a jax array and a callable is passed as one of
             # the arguments, since jax does not support computational graph explicitly.

From bf699542738372791fed015802d0cccae22f83cb Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 29 Dec 2024 19:03:16 -0500
Subject: [PATCH 160/170] update code

---
 deepxde/data/pde_operator.py  | 2 +-
 deepxde/gradients/jacobian.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index fb47dd803..75e08958b 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -278,7 +278,7 @@ def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
                 if not is_multi_outputs:
                     return output.reshape(shape0 * shape1, 1)
-                elif is_multi_outputs:
+                else:
                     return output.reshape(shape0 * shape1, shape2)
 
             if not is_multi_outputs:
diff --git a/deepxde/gradients/jacobian.py b/deepxde/gradients/jacobian.py
index d9e00fd55..4cbe46087 100644
--- a/deepxde/gradients/jacobian.py
+++ b/deepxde/gradients/jacobian.py
@@ -2,7 +2,6 @@
 
 from .. import config
 from ..backend import backend_name
-from .. import backend as bkd
 
 class Jacobian(ABC):
     """Compute `Jacobian matrix <https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant>`_

From 05fac47ac54770d74d7a4115dba4d4adb4355b19 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 29 Dec 2024 21:58:31 -0500
Subject: [PATCH 161/170] update code

---
 deepxde/data/pde_operator.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 75e08958b..cdc615fe4 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -276,10 +276,10 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
 
             def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
-                if not is_multi_outputs:
-                    return output.reshape(shape0 * shape1, 1)
-                else:
+                if is_multi_outputs:
                     return output.reshape(shape0 * shape1, shape2)
+                else:
+                    return output.reshape(shape0 * shape1, 1)
 
             if not is_multi_outputs:
                 outputs = outputs.reshape(shape0 * shape1, 1)

From 5f09216f1d95252310a959c2b29ff70c76bbed39 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Sun, 29 Dec 2024 22:02:31 -0500
Subject: [PATCH 162/170] update code

---
 deepxde/data/pde_operator.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index cdc615fe4..073191313 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -278,8 +278,7 @@ def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
                 if is_multi_outputs:
                     return output.reshape(shape0 * shape1, shape2)
-                else:
-                    return output.reshape(shape0 * shape1, 1)
+                return output.reshape(shape0 * shape1, 1)
 
             if not is_multi_outputs:
                 outputs = outputs.reshape(shape0 * shape1, 1)

From 06dda791d373aa25d1e7a5083b1f991cf5ac0036 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 10:30:12 -0500
Subject: [PATCH 163/170] update code

---
 deepxde/data/pde_operator.py  | 4 ++--
 deepxde/gradients/jacobian.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 073191313..1770389b3 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -263,10 +263,10 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward":  # forward mode AD
-            if bkd.ndim(outputs) == 2:
+            if model.net.num_outputs == 2:
                 is_multi_outputs = False
                 shape0, shape1 = outputs.shape[0], outputs.shape[1]
-            elif bkd.ndim(outputs) == 3:
+            elif model.net.num_outputs == 3:
                 is_multi_outputs = True
                 shape0, shape1, shape2 = (
                     outputs.shape[0],
diff --git a/deepxde/gradients/jacobian.py b/deepxde/gradients/jacobian.py
index 4cbe46087..745f3612d 100644
--- a/deepxde/gradients/jacobian.py
+++ b/deepxde/gradients/jacobian.py
@@ -27,7 +27,7 @@ def __init__(self, ys, xs):
             elif config.autodiff == "forward":
                 # For forward-mode AD, a tuple of a tensor and a callable is passed, 
                 # similar to backend jax.
-                self.dim_y = ys[0].shape[-1]
+                self.dim_y = ys[0].shape[1]
         elif backend_name == "jax":
             # For backend jax, a tuple of a jax array and a callable is passed as one of
             # the arguments, since jax does not support computational graph explicitly.

From 0c1c57b39082071d98637dae4887470c04b7cfe3 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 10:31:41 -0500
Subject: [PATCH 164/170] update code

---
 deepxde/data/pde_operator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index 1770389b3..ceb610956 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -263,10 +263,10 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward":  # forward mode AD
-            if model.net.num_outputs == 2:
+            if model.net.num_outputs == 1:
                 is_multi_outputs = False
                 shape0, shape1 = outputs.shape[0], outputs.shape[1]
-            elif model.net.num_outputs == 3:
+            elif model.net.num_outputs > 1:
                 is_multi_outputs = True
                 shape0, shape1, shape2 = (
                     outputs.shape[0],

From 0ab297316f75319ba6733a604c7730bdeaa4a4df Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 10:32:26 -0500
Subject: [PATCH 165/170] update code

---
 deepxde/gradients/jacobian.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepxde/gradients/jacobian.py b/deepxde/gradients/jacobian.py
index 745f3612d..5b0af016d 100644
--- a/deepxde/gradients/jacobian.py
+++ b/deepxde/gradients/jacobian.py
@@ -3,6 +3,7 @@
 from .. import config
 from ..backend import backend_name
 
+
 class Jacobian(ABC):
     """Compute `Jacobian matrix <https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant>`_
     J as J[i, j] = dy_i / dx_j, where i = 0, ..., dim_y - 1 and j = 0, ..., dim_x - 1.

From 20559b3081396e5abc1410ac5e9708503573a1b8 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 14:17:34 -0500
Subject: [PATCH 166/170] update code

---
 deepxde/gradients/gradients_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index cb2b9d008..c8c83e5ab 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -97,7 +97,7 @@ def grad_fn(x):
                 # also works for the output shape of (batch size 1, batch size 2, dim)
                 # such as multiple-output DeepONet.
                 self.J[i, j] = (
-                    self.J[j][0][..., i : i + 1],
+                    self.J[j][0][:, i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],
                 )
         return self.J[i, j]

From c2e4448c4c47b17d1b55dee043ca43c98399431f Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 14:18:47 -0500
Subject: [PATCH 167/170] update code

---
 deepxde/gradients/gradients_forward.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py
index c8c83e5ab..b58aa7621 100644
--- a/deepxde/gradients/gradients_forward.py
+++ b/deepxde/gradients/gradients_forward.py
@@ -92,10 +92,7 @@ def grad_fn(x):
                 # In backend tensorflow/pytorch/jax, a tuple of a tensor/tensor/array
                 # and a callable is returned, so that it is consistent with the argument,
                 # which is also a tuple. This is useful for further computation, e.g.,
-                # Hessian. The original code self.J[j][0][:, i : i + 1] is designed for
-                # the output shape of (batch size, dim), but we find that the code
-                # also works for the output shape of (batch size 1, batch size 2, dim)
-                # such as multiple-output DeepONet.
+                # Hessian.
                 self.J[i, j] = (
                     self.J[j][0][:, i : i + 1],
                     lambda x: self.J[j][1](x)[i : i + 1],

From 5d8ec8bcc54d4ab312431bdaf51b1e98b27c84bc Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 17:36:34 -0500
Subject: [PATCH 168/170] update code

---
 deepxde/data/pde_operator.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index ceb610956..e4cdbd24d 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -266,7 +266,7 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             if model.net.num_outputs == 1:
                 is_multi_outputs = False
                 shape0, shape1 = outputs.shape[0], outputs.shape[1]
-            elif model.net.num_outputs > 1:
+            else:
                 is_multi_outputs = True
                 shape0, shape1, shape2 = (
                     outputs.shape[0],
@@ -277,16 +277,18 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
                 if is_multi_outputs:
-                    return output.reshape(shape0 * shape1, shape2)
-                return output.reshape(shape0 * shape1, 1)
+                    return bkd.reshape(output, (shape0 * shape1, shape2))
+                return bkd.reshape(output, (shape0 * shape1, 1))
 
             if not is_multi_outputs:
-                outputs = outputs.reshape(shape0 * shape1, 1)
-                auxiliary_vars = model.net.auxiliary_vars.reshape(shape0 * shape1, 1)
-            elif is_multi_outputs:
-                outputs = outputs.reshape(shape0 * shape1, shape2)
-                auxiliary_vars = model.net.auxiliary_vars.reshape(
-                    shape0 * shape1, shape2
+                outputs = bkd.reshape(outputs, (shape0 * shape1, 1))
+                auxiliary_vars = bkd.reshape(
+                    model.net.auxiliary_vars, (shape0 * shape1, 1)
+                )
+            else:
+                outputs = bkd.reshape(outputs, (shape0 * shape1, shape2))
+                auxiliary_vars = bkd.reshape(
+                    model.net.auxiliary_vars, (shape0 * shape1, shape2)
                 )
 
             f = []
@@ -297,11 +299,11 @@ def forward_call(trunk_input):
                     f = [f]
 
             if not is_multi_outputs:
-                outputs = outputs.reshape(shape0, shape1)
-                f = [fi.reshape(shape0, shape1) for fi in f]
-            elif is_multi_outputs:
-                outputs = outputs.reshape(shape0, shape1, shape2)
-                f = [fi.reshape(shape0, shape1, shape2) for fi in f]
+                outputs = bkd.reshape(outputs, (shape0, shape1))
+                f = [bkd.reshape(fi, (shape0, shape1)) for fi in f]
+            else:
+                outputs = bkd.reshape(outputs, (shape0, shape1, shape2))
+                f = [bkd.reshape(fi, (shape0, shape1, shape2)) for fi in f]
             # Each error has the shape (N1, ~N2)
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
             for error in error_f:

From 7d36bb43d069291e2144307728e5e33557b9da38 Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 22:21:34 -0500
Subject: [PATCH 169/170] update code

---
 deepxde/data/pde_operator.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index e4cdbd24d..dbaf95db9 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -264,10 +264,8 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward":  # forward mode AD
             if model.net.num_outputs == 1:
-                is_multi_outputs = False
                 shape0, shape1 = outputs.shape[0], outputs.shape[1]
             else:
-                is_multi_outputs = True
                 shape0, shape1, shape2 = (
                     outputs.shape[0],
                     outputs.shape[1],
@@ -276,11 +274,11 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
 
             def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
-                if is_multi_outputs:
-                    return bkd.reshape(output, (shape0 * shape1, shape2))
-                return bkd.reshape(output, (shape0 * shape1, 1))
+                if model.net.num_outputs == 1:
+                    return bkd.reshape(output, (shape0 * shape1, 1))
+                return bkd.reshape(output, (shape0 * shape1, shape2))
 
-            if not is_multi_outputs:
+            if model.net.num_outputs == 1:
                 outputs = bkd.reshape(outputs, (shape0 * shape1, 1))
                 auxiliary_vars = bkd.reshape(
                     model.net.auxiliary_vars, (shape0 * shape1, 1)
@@ -298,7 +296,7 @@ def forward_call(trunk_input):
                 if not isinstance(f, (list, tuple)):
                     f = [f]
 
-            if not is_multi_outputs:
+            if model.net.num_outputs == 1:
                 outputs = bkd.reshape(outputs, (shape0, shape1))
                 f = [bkd.reshape(fi, (shape0, shape1)) for fi in f]
             else:

From 109a3af901b36f9087c18d687c88dc5d43b4d6fe Mon Sep 17 00:00:00 2001
From: Jerry-Jzy <jzyswufe@163.com>
Date: Tue, 31 Dec 2024 23:59:40 -0500
Subject: [PATCH 170/170] update code

---
 deepxde/data/pde_operator.py | 40 +++++++++++-------------------------
 1 file changed, 12 insertions(+), 28 deletions(-)

diff --git a/deepxde/data/pde_operator.py b/deepxde/data/pde_operator.py
index dbaf95db9..78c7d5393 100644
--- a/deepxde/data/pde_operator.py
+++ b/deepxde/data/pde_operator.py
@@ -263,45 +263,29 @@ def _losses(self, outputs, loss_fn, inputs, model, num_func, aux=None):
             # Use stack instead of as_tensor to keep the gradients.
             losses = [bkd.reduce_mean(bkd.stack(loss, 0)) for loss in losses]
         elif config.autodiff == "forward":  # forward mode AD
-            if model.net.num_outputs == 1:
-                shape0, shape1 = outputs.shape[0], outputs.shape[1]
-            else:
-                shape0, shape1, shape2 = (
-                    outputs.shape[0],
-                    outputs.shape[1],
-                    outputs.shape[2],
-                )
+            shape0, shape1 = outputs.shape[0], outputs.shape[1]
+            shape2 = 1 if model.net.num_outputs == 1 else outputs.shape[2]
 
             def forward_call(trunk_input):
                 output = aux[0]((inputs[0], trunk_input))
-                if model.net.num_outputs == 1:
-                    return bkd.reshape(output, (shape0 * shape1, 1))
                 return bkd.reshape(output, (shape0 * shape1, shape2))
 
-            if model.net.num_outputs == 1:
-                outputs = bkd.reshape(outputs, (shape0 * shape1, 1))
-                auxiliary_vars = bkd.reshape(
-                    model.net.auxiliary_vars, (shape0 * shape1, 1)
-                )
-            else:
-                outputs = bkd.reshape(outputs, (shape0 * shape1, shape2))
-                auxiliary_vars = bkd.reshape(
-                    model.net.auxiliary_vars, (shape0 * shape1, shape2)
-                )
-
             f = []
             if self.pde.pde is not None:
                 # Each f has the shape (N1, N2)
-                f = self.pde.pde(inputs[1], (outputs, forward_call), auxiliary_vars)
+                f = self.pde.pde(
+                    inputs[1],
+                    (bkd.reshape(outputs, (shape0 * shape1, shape2)), forward_call),
+                    bkd.reshape(model.net.auxiliary_vars, (shape0 * shape1, shape2)),
+                )
                 if not isinstance(f, (list, tuple)):
                     f = [f]
+            f = (
+                [bkd.reshape(fi, (shape0, shape1)) for fi in f]
+                if model.net.num_outputs == 1
+                else [bkd.reshape(fi, (shape0, shape1, shape2)) for fi in f]
+            )
 
-            if model.net.num_outputs == 1:
-                outputs = bkd.reshape(outputs, (shape0, shape1))
-                f = [bkd.reshape(fi, (shape0, shape1)) for fi in f]
-            else:
-                outputs = bkd.reshape(outputs, (shape0, shape1, shape2))
-                f = [bkd.reshape(fi, (shape0, shape1, shape2)) for fi in f]
             # Each error has the shape (N1, ~N2)
             error_f = [fi[:, bcs_start[-1] :] for fi in f]
             for error in error_f: