From 2a76066a6545d429029d1bfa01b0349849a433bc Mon Sep 17 00:00:00 2001 From: Chan Lee <150145948+ChanLumerico@users.noreply.github.com> Date: Sat, 10 Aug 2024 09:08:37 +0000 Subject: [PATCH 1/3] BasicBlock of resnet --- luma/neural/_specials/resnet.py | 30 ++++++++++++++++++++++++++++-- luma/neural/block.py | 9 +++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/luma/neural/_specials/resnet.py b/luma/neural/_specials/resnet.py index ea172f2..61900c1 100644 --- a/luma/neural/_specials/resnet.py +++ b/luma/neural/_specials/resnet.py @@ -15,6 +15,7 @@ def __init__( in_channels: int, out_channels: int, stride: int = 1, + downsampling: LayerLike | None = None, activation: Activation.FuncType = Activation.ReLU, optimizer: Optimizer | None = None, initializer: InitUtil.InitStr = None, @@ -26,6 +27,7 @@ def __init__( self.in_channels = in_channels self.out_channels = out_channels self.stride = stride + self.downsampling = downsampling self.activation = activation self.optimizer = optimizer self.initializer = initializer @@ -42,8 +44,9 @@ def __init__( self.init_nodes() super(_Basic, self).__init__( graph={ - self.rt_: [self.sum_, self.conv_], + self.rt_: [self.down_, self.conv_], self.conv_: [self.sum_], + self.down_: [self.sum_], }, root=self.rt_, term=self.sum_, @@ -77,10 +80,33 @@ def init_nodes(self) -> None: ), name="conv_", ) + self.down_ = LayerNode( + self.downsampling if self.downsampling else Identity(), + name="down_", + ) self.sum_ = LayerNode( self.activation(), merge_mode="sum", name="sum_", ) - ... + @Tensor.force_dim(4) + def forward(self, X: TensorLike, is_train: bool = False) -> TensorLike: + return super().forward(X, is_train) + + @Tensor.force_dim(4) + def backward(self, d_out: TensorLike) -> TensorLike: + return super().backward(d_out) + + @override + def out_shape(self, in_shape: Tuple[int]) -> Tuple[int]: + batch_size, _, height, width = in_shape + if self.downsampling: + _, _, down_h, down_w = self.downsampling.out_shape(in_shape) + return batch_size, self.out_channels, down_h, down_w + + return batch_size, self.out_channels, height, width + + +class _Bottleneck(LayerGraph): + NotImplemented diff --git a/luma/neural/block.py b/luma/neural/block.py index bc1d533..a1b3a30 100644 --- a/luma/neural/block.py +++ b/luma/neural/block.py @@ -17,6 +17,7 @@ "DenseBlock", "IncepBlock", "IncepResBlock", + "ResNetBlock", ) @@ -884,3 +885,11 @@ class V2_Redux(_specials.incep_res_v2._IncepRes_V2_Redux): Output: Tensor[-1, 2272, 8, 8] ``` """ + + +@ClassType.non_instantiable() +class ResNetBlock: + + class Basic(_specials.resnet._Basic): ... + + class Bottleneck(_specials.resnet._Bottleneck): ... From d55cdbf818afa03e88ffc0cf06dfeedd03aaafe4 Mon Sep 17 00:00:00 2001 From: Chan Lee <150145948+ChanLumerico@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:51:50 +0000 Subject: [PATCH 2/3] Bottleneck and @force_shape on _models/ --- luma/neural/_models/alex.py | 16 +++-- luma/neural/_models/incep.py | 48 ++++++++----- luma/neural/_models/lenet.py | 24 ++++--- luma/neural/_models/vgg.py | 32 +++++---- luma/neural/_specials/resnet.py | 116 ++++++++++++++++++++++++++++++-- 5 files changed, 187 insertions(+), 49 deletions(-) diff --git a/luma/neural/_models/alex.py b/luma/neural/_models/alex.py index b01722e..bcddc43 100644 --- a/luma/neural/_models/alex.py +++ b/luma/neural/_models/alex.py @@ -204,18 +204,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 3, 227, 227) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_AlexNet, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_AlexNet, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -410,18 +412,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 3, 227, 227) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_ZFNet, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_ZFNet, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, diff --git a/luma/neural/_models/incep.py b/luma/neural/_models/incep.py index a2ba3e8..b6807f6 100644 --- a/luma/neural/_models/incep.py +++ b/luma/neural/_models/incep.py @@ -179,18 +179,20 @@ def build_model(self) -> None: self.model += Flatten() self.model += Dense(1024, self.out_features, **base_args) + + input_shape: tuple = (-1, 3, 224, 224) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_Inception_V1, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_Inception_V1, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -383,18 +385,20 @@ def build_model(self) -> None: Dropout(self.dropout_rate, self.random_state), Dense(2048, self.out_features, **base_args), ) + + input_shape: tuple = (-1, 3, 299, 299) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_Inception_V2, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_Inception_V2, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -596,20 +600,22 @@ def build_model(self) -> None: Dropout(self.dropout_rate, self.random_state), Dense(2048, self.out_features, **base_args), ) + + input_shape: tuple = (-1, 3, 299, 299) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: ls = LabelSmoothing(smoothing=self.smoothing) y_ls = ls.fit_transform(y) return super(_Inception_V3, self).fit_nn(X, y_ls) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_Inception_V3, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -720,20 +726,22 @@ def build_model(self) -> None: Dropout(self.dropout_rate, self.random_state), Dense(1536, self.out_features), ) + + input_shape: tuple = (-1, 3, 299, 299) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: ls = LabelSmoothing(smoothing=self.smoothing) y_ls = ls.fit_transform(y) return super(_Inception_V4, self).fit_nn(X, y_ls) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_Inception_V4, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -843,20 +851,22 @@ def build_model(self) -> None: Dropout(self.dropout_rate, self.random_state), Dense(1792, self.out_features), ) + + input_shape: tuple = (-1, 3, 299, 299) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: ls = LabelSmoothing(smoothing=self.smoothing) y_ls = ls.fit_transform(y) return super(_InceptionRes_V1, self).fit_nn(X, y_ls) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_InceptionRes_V1, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -969,20 +979,22 @@ def build_model(self) -> None: Dropout(self.dropout_rate, self.random_state), Dense(2272, self.out_features), ) + + input_shape: tuple = (-1, 3, 299, 299) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: ls = LabelSmoothing(smoothing=self.smoothing) y_ls = ls.fit_transform(y) return super(_InceptionRes_V2, self).fit_nn(X, y_ls) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_InceptionRes_V2, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, diff --git a/luma/neural/_models/lenet.py b/luma/neural/_models/lenet.py index adaca1c..ba37f07 100644 --- a/luma/neural/_models/lenet.py +++ b/luma/neural/_models/lenet.py @@ -112,18 +112,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 1, 28, 28) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_LeNet_1, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_LeNet_1, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -244,18 +246,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 1, 32, 32) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_LeNet_4, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_LeNet_4, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -387,17 +391,19 @@ def build_model(self) -> None: random_state=self.random_state, ) - @Tensor.force_dim(4) + input_shape: tuple = (-1, 1, 32, 32) + + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_LeNet_5, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_LeNet_5, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, diff --git a/luma/neural/_models/vgg.py b/luma/neural/_models/vgg.py index e03f4a6..bf60d2d 100644 --- a/luma/neural/_models/vgg.py +++ b/luma/neural/_models/vgg.py @@ -167,18 +167,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 3, 224, 224) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_VGGNet_11, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_VGGNet_11, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -344,18 +346,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 3, 224, 224) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_VGGNet_13, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_VGGNet_13, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -534,17 +538,19 @@ def build_model(self) -> None: random_state=self.random_state, ) - @Tensor.force_dim(4) + input_shape: tuple = (-1, 3, 224, 224) + + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_VGGNet_16, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_VGGNet_16, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, @@ -734,18 +740,20 @@ def build_model(self) -> None: lambda_=self.lambda_, random_state=self.random_state, ) + + input_shape: tuple = (-1, 3, 224, 224) - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def fit(self, X: Tensor, y: Matrix) -> Self: return super(_VGGNet_19, self).fit_nn(X, y) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: return super(_VGGNet_19, self).predict_nn(X, argmax) @override - @Tensor.force_dim(4) + @Tensor.force_shape(input_shape) def score( self, X: Tensor, diff --git a/luma/neural/_specials/resnet.py b/luma/neural/_specials/resnet.py index 61900c1..67b4a52 100644 --- a/luma/neural/_specials/resnet.py +++ b/luma/neural/_specials/resnet.py @@ -102,11 +102,119 @@ def backward(self, d_out: TensorLike) -> TensorLike: def out_shape(self, in_shape: Tuple[int]) -> Tuple[int]: batch_size, _, height, width = in_shape if self.downsampling: - _, _, down_h, down_w = self.downsampling.out_shape(in_shape) - return batch_size, self.out_channels, down_h, down_w - + _, _, height, width = self.downsampling.out_shape(in_shape) + return batch_size, self.out_channels, height, width class _Bottleneck(LayerGraph): - NotImplemented + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + expansion: int = 4, + downsampling: LayerLike | None = None, + activation: Activation.FuncType = Activation.ReLU, + optimizer: Optimizer | None = None, + initializer: InitUtil.InitStr = None, + lambda_: float = 0.0, + do_batch_norm: bool = True, + momentum: float = 0.9, + random_state: int | None = None, + ) -> None: + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.expansion = expansion + self.downsampling = downsampling + self.activation = activation + self.optimizer = optimizer + self.initializer = initializer + self.lambda_ = lambda_ + self.do_batch_norm = do_batch_norm + self.momentum = momentum + + self.basic_args = { + "initializer": initializer, + "lambda_": lambda_, + "random_state": random_state, + } + + self.init_nodes() + super(_Bottleneck, self).__init__( + graph={ + self.rt_: [self.down_, self.conv_], + self.conv_: [self.sum_], + self.down_: [self.sum_], + }, + root=self.rt_, + term=self.sum_, + ) + + def init_nodes(self) -> None: + self.rt_ = LayerNode(Identity(), name="rt_") + self.conv_ = LayerNode( + Sequential( + Convolution2D( + self.in_channels, + self.out_channels, + 1, + self.stride, + **self.basic_args + ), + BatchNorm2D(self.out_channels, self.momentum), + self.activation(), + Convolution2D( + self.out_channels, + self.out_channels, + 3, + self.stride, + **self.basic_args + ), + BatchNorm2D(self.out_channels, self.momentum), + self.activation(), + Convolution2D( + self.out_channels, + self.out_channels * self.expansion, + 1, + self.stride, + **self.basic_args + ), + BatchNorm2D( + self.out_channels * self.expansion, + self.momentum, + ), + ), + name="conv_", + ) + self.down_ = LayerNode( + self.downsampling if self.downsampling else Identity(), + name="down_", + ) + self.sum_ = LayerNode( + self.activation(), + merge_mode="sum", + name="sum_", + ) + + @Tensor.force_dim(4) + def forward(self, X: TensorLike, is_train: bool = False) -> TensorLike: + return super().forward(X, is_train) + + @Tensor.force_dim(4) + def backward(self, d_out: TensorLike) -> TensorLike: + return super().backward(d_out) + + @override + def out_shape(self, in_shape: Tuple[int]) -> Tuple[int]: + batch_size, _, height, width = in_shape + if self.downsampling: + _, _, height, width = self.downsampling.out_shape(in_shape) + + return ( + batch_size, + self.out_channels * self.expansion, + height, + width, + ) From f3ecd4e4b28f1a5c250ee966af5b52d15e22b213 Mon Sep 17 00:00:00 2001 From: Chan Lee <150145948+ChanLumerico@users.noreply.github.com> Date: Sat, 10 Aug 2024 20:19:50 +0900 Subject: [PATCH 3/3] ResNetBlock docstrings --- luma/neural/block.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/luma/neural/block.py b/luma/neural/block.py index a1b3a30..608d2ab 100644 --- a/luma/neural/block.py +++ b/luma/neural/block.py @@ -724,7 +724,7 @@ class IncepResBlock: References ---------- - Inception-ResNet V1, V2 : + `Inception-ResNet V1, V2` : [1] Szegedy, Christian, et al. “Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning.” Proceedings of the Thirty-First AAAI Conference on @@ -889,7 +889,42 @@ class V2_Redux(_specials.incep_res_v2._IncepRes_V2_Redux): @ClassType.non_instantiable() class ResNetBlock: + """ + Container class for building components of ResNet. + + References + ---------- + `ResNet-(18, 34, 50, 101, 152)` : + [1] He, Kaiming, et al. “Deep Residual Learning for Image + Recognition.” Proceedings of the IEEE Conference on Computer + Vision and Pattern Recognition (CVPR), 2016, pp. 770-778. - class Basic(_specials.resnet._Basic): ... + """ - class Bottleneck(_specials.resnet._Bottleneck): ... + class Basic(_specials.resnet._Basic): + """ + Basic convolution block used in `ResNet-18` and `ResNet-34`. + + Parameters + ---------- + `downsampling` : LayerLike, optional + An additional layer to the input signal which reduces + its grid size to perform a downsampling + + See [1] also for additional information. + """ + + class Bottleneck(_specials.resnet._Bottleneck): + """ + Bottleneck block used in `ResNet-(50, 101, 152)`. + + Parameters + ---------- + `downsampling` : LayerLike, optional + An additional layer to the input signal which reduces + its grid size to perform a downsampling + `expansion` : int, default=4 + Expanding factor for the number of output channels. + + See [1] also for additional information. + """