From 755b1ec0857e4f263b17aabe795cf8ca81ab6159 Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Mon, 26 Aug 2024 23:47:27 +0900 Subject: [PATCH] autoprop/ --- luma/neural/autoprop/__init__.py | 8 ++ .../neural/{autoprop.py => autoprop/graph.py} | 30 ++----- luma/neural/autoprop/merge.py | 84 +++++++++++++++++++ luma/neural/model/__init__.py | 2 +- 4 files changed, 102 insertions(+), 22 deletions(-) create mode 100644 luma/neural/autoprop/__init__.py rename luma/neural/{autoprop.py => autoprop/graph.py} (93%) create mode 100644 luma/neural/autoprop/merge.py diff --git a/luma/neural/autoprop/__init__.py b/luma/neural/autoprop/__init__.py new file mode 100644 index 0000000..2925fb9 --- /dev/null +++ b/luma/neural/autoprop/__init__.py @@ -0,0 +1,8 @@ +""" +`AutoProp` +---------- +Auto propagation system for complex neural networks of Luma Python library. + +""" + +from .graph import LayerNode, LayerGraph diff --git a/luma/neural/autoprop.py b/luma/neural/autoprop/graph.py similarity index 93% rename from luma/neural/autoprop.py rename to luma/neural/autoprop/graph.py index 100ca6b..6fc872b 100644 --- a/luma/neural/autoprop.py +++ b/luma/neural/autoprop/graph.py @@ -1,12 +1,14 @@ -from typing import List, Literal, Dict, Self, Any, Tuple +from typing import List, Dict, Self, Any, Tuple from collections import deque import numpy as np from luma.interface.typing import TensorLike, LayerLike -from luma.interface.exception import NotFittedError from luma.interface.util import Clone +from luma.interface.exception import NotFittedError from luma.core.super import Optimizer +from .merge import MergeMode + __all__ = ("LayerNode", "LayerGraph") @@ -15,7 +17,7 @@ class LayerNode: def __init__( self, layer: LayerLike, - merge_mode: Literal["chcat", "sum"] = "sum", + merge_mode: MergeMode = MergeMode.SUM, name: str | None = None, ) -> None: self.layer: LayerLike = layer @@ -42,13 +44,8 @@ def back_enqueue(self, d_out: TensorLike) -> None: self.b_queue.append(d_out) def forward(self, is_train: bool = False) -> TensorLike: - match self.merge_mode: - case "chcat": - X = np.concatenate(self.f_queue, axis=1) - case "sum": - X = np.sum(self.f_queue, axis=0) - out = self.layer(X, is_train) - return out + X = self.merge_mode.forward(self.f_queue) + return self.layer(X, is_train) def backward(self) -> List[TensorLike]: d_cum = np.sum(self.b_queue, axis=0) @@ -57,17 +54,8 @@ def backward(self) -> List[TensorLike]: return [d_out] d_out_arr = [] - for i in range(self.n_backward): - if self.merge_mode == "chcat": - d_out_arr.append( - d_out[ - :, - self.cum_ch[i] : self.cum_ch[i + 1], - ..., - ] - ) - elif self.merge_mode == "sum": - d_out_arr.append(d_out) + for i in range(self.n_forward): + d_out_arr.append(self.merge_mode.backward(self.f_queue, d_out, i)) return d_out_arr diff --git a/luma/neural/autoprop/merge.py b/luma/neural/autoprop/merge.py new file mode 100644 index 0000000..77bb265 --- /dev/null +++ b/luma/neural/autoprop/merge.py @@ -0,0 +1,84 @@ +from enum import Enum +import numpy as np + +from luma.interface.typing import TensorLike + + +class MergeMode(Enum): + CHCAT = "chcat" + SUM = "sum" + HADAMARD = "hadamard" + AVERAGE = "average" + MAX = "max" + MIN = "min" + DOT = "dot" + SUBTRACT = "subtract" + + def forward(self, f_queue: list[TensorLike]) -> TensorLike: + match self: + case MergeMode.CHCAT: + return np.concatenate(f_queue, axis=1) + + case MergeMode.SUM: + return np.sum(f_queue, axis=0) + + case MergeMode.HADAMARD: + X = np.ones_like(f_queue[0]) + for tensor in f_queue: + X *= tensor + return X + + case MergeMode.AVERAGE: + return np.mean(f_queue, axis=0) + + case MergeMode.MAX: + return np.maximum.reduce(f_queue) + + case MergeMode.MIN: + return np.minimum.reduce(f_queue) + + case MergeMode.DOT: + return np.dot(f_queue[0], f_queue[1]) + + case MergeMode.SUBTRACT: + result = f_queue[0] + for tensor in f_queue[1:]: + result -= tensor + return result + + def backward( + self, f_queue: list[TensorLike], d_out: TensorLike, i: int + ) -> TensorLike: + match self: + case MergeMode.CHCAT: + cum_ch = [0] + for tensor in f_queue: + cum_ch.append(cum_ch[-1] + tensor.shape[1]) + return d_out[:, cum_ch[i] : cum_ch[i + 1], ...] + + case MergeMode.SUM: + return d_out + + case MergeMode.HADAMARD: + prod_except_current = np.ones_like(f_queue[0]) + for j in range(len(f_queue)): + if j != i: + prod_except_current *= f_queue[j] + return d_out * prod_except_current + + case MergeMode.AVERAGE: + return d_out / len(f_queue) + + case MergeMode.MAX | MergeMode.MIN: + return ( + d_out * (f_queue[i] == getattr(np, self.value).reduce(f_queue)) + ).astype(d_out.dtype) + + case MergeMode.DOT: + if i == 0: + return np.dot(d_out, f_queue[1].T) + elif i == 1: + return np.dot(f_queue[0].T, d_out) + + case MergeMode.SUBTRACT: + return d_out if i == 0 else -d_out diff --git a/luma/neural/model/__init__.py b/luma/neural/model/__init__.py index a2f00e0..f77bad6 100644 --- a/luma/neural/model/__init__.py +++ b/luma/neural/model/__init__.py @@ -2989,7 +2989,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super(MobileNet_V1, self).__init__( + super(MobileNet_V2, self).__init__( activation, initializer, out_features,