From 44890e9aefc367b1ffd8d450227848eb0a07e4bb Mon Sep 17 00:00:00 2001 From: Chan Lee <150145948+ChanLumerico@users.noreply.github.com> Date: Sun, 11 Aug 2024 09:57:39 +0000 Subject: [PATCH] ResNet_34 --- luma/__import__.py | 5 +- luma/neural/_models/resnet.py | 145 ++++++++++++++++++++++++++++++++-- luma/neural/model.py | 138 +++++++++++++++++++++++++++----- 3 files changed, 262 insertions(+), 26 deletions(-) diff --git a/luma/__import__.py b/luma/__import__.py index edb45b7..f8b2f9d 100644 --- a/luma/__import__.py +++ b/luma/__import__.py @@ -151,6 +151,8 @@ Inception_V4, InceptionResNet_V1, InceptionResNet_V2, + ResNet_18, + ResNet_34, ) from luma.neural.autoprop import LayerNode, LayerGraph @@ -333,7 +335,8 @@ AlexNet, ZFNet, VGGNet_11, VGGNet_13, VGGNet_16, VGGNet_19, Inception_V1, Inception_V2, Inception_V3, Inception_V4, - InceptionResNet_V1, InceptionResNet_V2 + InceptionResNet_V1, InceptionResNet_V2, + ResNet_18, ResNet_34, # ------------------- [ luma.metric ] ---------------------- Accuracy, Precision, Recall, F1Score, Specificity diff --git a/luma/neural/_models/resnet.py b/luma/neural/_models/resnet.py index fcf32a8..c52360f 100644 --- a/luma/neural/_models/resnet.py +++ b/luma/neural/_models/resnet.py @@ -1,8 +1,8 @@ from typing import Any, Self, override, List, Optional from dataclasses import asdict, dataclass -from luma.core.super import Estimator, Evaluator, Optimizer, Supervised -from luma.interface.typing import Matrix, Tensor, TensorLike, Vector +from luma.core.super import Estimator, Evaluator, Supervised +from luma.interface.typing import Matrix, Tensor, Vector from luma.interface.util import InitUtil from luma.metric.classification import Accuracy @@ -107,10 +107,10 @@ def __init__( self.feature_sizes_ = [ [3, 64], - [64, 64, 64, 64], - [128, 128, 128, 128], - [256, 256, 256, 256], - [512, 512, 512, 512], + [64, 64] * 2, + [128, 128] * 2, + [256, 256] * 2, + [512, 512] * 2, ] self.feature_shapes_ = [ self._get_feature_shapes(sizes) for sizes in self.feature_sizes_ @@ -197,3 +197,136 @@ def score( argmax: bool = True, ) -> float: return super(_ResNet_18, self).score_nn(X, y, metric, argmax) + + +class _ResNet_34(Estimator, Supervised, NeuralModel): + def __init__( + self, + activation: Activation.FuncType = Activation.ReLU, + initializer: InitUtil.InitStr = None, + out_features: int = 1000, + batch_size: int = 128, + n_epochs: int = 100, + valid_size: float = 0.1, + lambda_: float = 0.0, + momentum: float = 0.9, + early_stopping: bool = False, + patience: int = 10, + shuffle: bool = True, + random_state: int | None = None, + deep_verbose: bool = False, + ) -> None: + self.activation = activation + self.initializer = initializer + self.out_features = out_features + self.lambda_ = lambda_ + self.momentum = momentum + self.shuffle = shuffle + self.random_state = random_state + self._fitted = False + + super().__init__( + batch_size, + n_epochs, + valid_size, + early_stopping, + patience, + shuffle, + random_state, + deep_verbose, + ) + super().init_model() + self.model = Sequential() + + self.feature_sizes_ = [ + [3, 64], + [64, 64] * 3, + [128, 128] * 4, + [256, 256] * 6, + [512, 512] * 3, + ] + self.feature_shapes_ = [ + self._get_feature_shapes(sizes) for sizes in self.feature_sizes_ + ] + + self.set_param_ranges( + { + "out_features": ("0<,+inf", int), + "batch_size": ("0<,+inf", int), + "n_epochs": ("0<,+inf", int), + "valid_size": ("0<,<1", None), + "momentum": ("0,1", None), + "dropout_rate": ("0,1", None), + "lambda_": ("0,+inf", None), + "patience": ("0<,+inf", int), + } + ) + self.check_param_ranges() + self.build_model() + + def build_model(self) -> None: + base_args = { + "initializer": self.initializer, + "lambda_": self.lambda_, + "random_state": self.random_state, + } + res_args = BaseBlockArgs( + activation=self.activation, + do_batch_norm=True, + momentum=self.momentum, + **base_args, + ) + + self.model.extend( + Convolution2D(3, 64, 7, 2, 3, **base_args), + BatchNorm2D(64, self.momentum), + self.activation(), + Pooling2D(3, 2, "max", "same"), + ) + self.layer_2, in_channels = _make_layer( + 64, 64, BasicBlock, 3, 2, base_args, res_args + ) + self.layer_3, in_channels = _make_layer( + in_channels, 128, BasicBlock, 4, 3, base_args, res_args, stride=2 + ) + self.layer_4, in_channels = _make_layer( + in_channels, 256, BasicBlock, 6, 4, base_args, res_args, stride=2 + ) + self.layer_5, in_channels = _make_layer( + in_channels, 512, BasicBlock, 3, 5, base_args, res_args, stride=2 + ) + + self.model.extend( + self.layer_2, + self.layer_3, + self.layer_4, + self.layer_5, + deep_add=True, + ) + self.model.extend( + AdaptiveAvgPooling2D((1, 1)), + Flatten(), + Dense(512 * BasicBlock.expansion, self.out_features, **base_args), + ) + + input_shape: tuple = (-1, 3, 224, 224) + + @Tensor.force_shape(input_shape) + def fit(self, X: Tensor, y: Matrix) -> Self: + return super(_ResNet_34, self).fit_nn(X, y) + + @override + @Tensor.force_shape(input_shape) + def predict(self, X: Tensor, argmax: bool = True) -> Matrix | Vector: + return super(_ResNet_34, self).predict_nn(X, argmax) + + @override + @Tensor.force_shape(input_shape) + def score( + self, + X: Tensor, + y: Matrix, + metric: Evaluator = Accuracy, + argmax: bool = True, + ) -> float: + return super(_ResNet_34, self).score_nn(X, y, metric, argmax) diff --git a/luma/neural/model.py b/luma/neural/model.py index f9b3bb0..f8cb117 100644 --- a/luma/neural/model.py +++ b/luma/neural/model.py @@ -25,6 +25,7 @@ "InceptionResNet_V1", "InceptionResNet_V2", "ResNet_18", + "ResNet_34", ) @@ -103,7 +104,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False ) -> None: - super().__init__( + super(SimpleMLP, self).__init__( in_features, out_features, hidden_layers, @@ -259,7 +260,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False ) -> None: - super().__init__( + super(SimpleCNN, self).__init__( in_channels_list, in_features_list, out_channels, @@ -368,7 +369,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(LeNet_1, self).__init__( activation, initializer, out_features, @@ -458,7 +459,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(LeNet_4, self).__init__( activation, initializer, out_features, @@ -551,7 +552,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(LeNet_5, self).__init__( activation, initializer, out_features, @@ -653,7 +654,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(AlexNet, self).__init__( activation, initializer, out_features, @@ -755,7 +756,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(ZFNet, self).__init__( activation, initializer, out_features, @@ -859,7 +860,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(VGGNet_11, self).__init__( activation, initializer, out_features, @@ -966,7 +967,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(VGGNet_13, self).__init__( activation, initializer, out_features, @@ -1076,7 +1077,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(VGGNet_16, self).__init__( activation, initializer, out_features, @@ -1189,7 +1190,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(VGGNet_19, self).__init__( activation, initializer, out_features, @@ -1303,7 +1304,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(Inception_V1, self).__init__( activation, initializer, out_features, @@ -1415,7 +1416,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(Inception_V2, self).__init__( activation, initializer, out_features, @@ -1531,7 +1532,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(Inception_V3, self).__init__( activation, initializer, out_features, @@ -1638,7 +1639,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(Inception_V4, self).__init__( activation, initializer, out_features, @@ -1740,7 +1741,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(InceptionResNet_V1, self).__init__( activation, initializer, out_features, @@ -1841,7 +1842,7 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(InceptionResNet_V2, self).__init__( activation, initializer, out_features, @@ -1861,7 +1862,7 @@ def __init__( class ResNet_18(_models.resnet._ResNet_18): """ - ResNet18 is a 18-layer deep neural network that uses residual blocks + ResNet-18 is a 18-layer deep neural network that uses residual blocks to improve training by learning residuals, helping prevent vanishing gradients and enabling better performance in image recognition tasks. @@ -1941,7 +1942,106 @@ def __init__( random_state: int | None = None, deep_verbose: bool = False, ) -> None: - super().__init__( + super(ResNet_18, self).__init__( + activation, + initializer, + out_features, + batch_size, + n_epochs, + valid_size, + lambda_, + momentum, + early_stopping, + patience, + shuffle, + random_state, + deep_verbose, + ) + + +class ResNet_34(_models.resnet._ResNet_34): + """ + ResNet-34 is a 34-layer deep neural network that uses residual blocks + to improve training by learning residuals, helping prevent vanishing + gradients and enabling better performance in image recognition tasks. + + Structure + --------- + Input: + ```py + Tensor[..., 3, 224, 224] + ``` + Residual Blocks: + ```py + Convolution2D(3, 64, filter_size=7, stride=2) -> # conv1 + + 3x ResNetBlock.Basic(64, 64) -> # conv2 + 4x ResNetBlock.Basic(128, 128, stride=2) -> # conv3 + 6x ResNetBlock.Basic(256, 256, stride=2) -> # conv4 + 3x ResNetBlock.Basic(512, 512, stride=2) -> # conv5 + + AdaptiveAvgPooling2D((1, 1)) -> # avg pool + ``` + Fully Connected Layers: + ```py + Flatten -> Dense(512, 1000) + ``` + Output: + ```py + Matrix[..., 1000] + ``` + Parameter Size: + ```txt + 21,796,672 weights, 9,512 biases -> 21,806,184 params + ``` + Parameters + ---------- + `activation` : FuncType, default=Activation.ReLU + Type of activation function + `initializer` : InitStr, default=None + Type of weight initializer + `out_features` : int, default=1000 + Number of output features + `batch_size` : int, default=100 + Size of a single mini-batch + `n_epochs` : int, default=100 + Number of epochs for training + `valid_size` : float, default=0.1 + Fractional size of validation set + `lambda_` : float, default=0.0 + L2 regularization strength + `early_stopping` : bool, default=False + Whether to early-stop the training when the valid score stagnates + `patience` : int, default=10 + Number of epochs to wait until early-stopping + `shuffle` : bool, default=True + Whethter to shuffle the data at the beginning of every epoch + + References + ---------- + 1. He, Kaiming, et al. “Deep Residual Learning for Image Recognition.” + Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition + (CVPR), 2016, pp. 770-778. + + """ + + def __init__( + self, + activation: Activation.FuncType = Activation.ReLU, + initializer: InitUtil.InitStr = None, + out_features: int = 1000, + batch_size: int = 128, + n_epochs: int = 100, + valid_size: float = 0.1, + lambda_: float = 0, + momentum: float = 0.9, + early_stopping: bool = False, + patience: int = 10, + shuffle: bool = True, + random_state: int | None = None, + deep_verbose: bool = False, + ) -> None: + super(ResNet_34, self).__init__( activation, initializer, out_features,