From 19be94a5913e25126f56a7d4898b9f83c3d88043 Mon Sep 17 00:00:00 2001 From: ksandor Date: Thu, 27 Jun 2024 15:47:31 +0200 Subject: [PATCH 1/7] 162 | Restructured folders --- epochalyst/_core/__init__.py | 1 - epochalyst/_core/_caching/__init__.py | 1 - epochalyst/_core/_pipeline/__init__.py | 1 - epochalyst/logging/__init__.py | 1 - epochalyst/{pipeline => }/model/__init__.py | 0 .../_custom_data_parallel.py | 0 epochalyst/{pipeline => model}/ensemble.py | 2 +- epochalyst/{pipeline => }/model/model.py | 5 +---- .../{pipeline => }/model/training/__init__.py | 0 .../model/training/augmentation/__init__.py | 0 .../augmentation/image_augmentations.py | 0 .../augmentation/time_series_augmentations.py | 2 +- .../model/training/augmentation/utils.py | 2 +- .../model/training/models/__init__.py | 0 .../model/training/models/timm.py | 0 .../model/training/pretrain_block.py | 2 +- .../model/training/torch_trainer.py | 8 ++++---- .../{pipeline => }/model/training/training.py | 16 ++++++++-------- .../model/training/training_block.py | 4 ++-- .../model/training/utils/__init__.py | 0 .../model/training/utils/get_dependencies.py | 0 .../model/training/utils/recursive_repr.py | 0 .../model/training/utils/tensor_functions.py | 0 .../model/transformation/__init__.py | 0 .../model/transformation/transformation.py | 13 ++++++------- .../transformation/transformation_block.py | 8 ++++---- epochalyst/pipeline/__init__.py | 1 - epochalyst/utils/__init__.py | 0 .../_caching/_cacher.py => utils/cacher.py} | 18 ++++++++++-------- epochalyst/{logging => utils}/logger.py | 5 +---- tests/model/__init__.py | 0 tests/{pipeline => model}/test_ensemble.py | 10 ++++------ tests/{pipeline => }/model/test_model.py | 8 +++----- .../test_recursive_repr.py | 6 ++---- tests/model/training/__init__.py | 0 tests/model/training/augmentation/__init__.py | 0 .../augmentation/test_audio/white_noise.wav | Bin .../augmentation/test_image_augmentations.py | 2 +- .../test_time_series_augmentations.py | 4 ++-- .../model/training/augmentation/test_utils.py | 2 +- tests/model/training/models/__init__.py | 0 .../model/training/models/test_timm.py | 2 +- .../model/training/test_pretrain_block.py | 2 +- .../model/training/test_torch_trainer.py | 8 +++----- .../model/training/test_training.py | 6 ++---- .../model/training/test_training_block.py | 3 +-- tests/model/training/utils/__init__.py | 0 .../training/utils/test_get_dependencies.py | 2 +- tests/model/transformation/__init__.py | 0 .../transformation/test_transformation.py | 6 ++---- .../test_transformation_block.py | 3 +-- tests/utils/__init__.py | 0 .../{_core/_caching => utils}/test__cacher.py | 6 +++--- tests/{logging => utils}/test_logger.py | 2 +- 54 files changed, 69 insertions(+), 93 deletions(-) delete mode 100644 epochalyst/_core/__init__.py delete mode 100644 epochalyst/_core/_caching/__init__.py delete mode 100644 epochalyst/_core/_pipeline/__init__.py delete mode 100644 epochalyst/logging/__init__.py rename epochalyst/{pipeline => }/model/__init__.py (100%) rename epochalyst/{_core/_pipeline => model}/_custom_data_parallel.py (100%) rename epochalyst/{pipeline => model}/ensemble.py (96%) rename epochalyst/{pipeline => }/model/model.py (92%) rename epochalyst/{pipeline => }/model/training/__init__.py (100%) rename epochalyst/{pipeline => }/model/training/augmentation/__init__.py (100%) rename epochalyst/{pipeline => }/model/training/augmentation/image_augmentations.py (100%) rename epochalyst/{pipeline => }/model/training/augmentation/time_series_augmentations.py (99%) rename epochalyst/{pipeline => }/model/training/augmentation/utils.py (98%) rename epochalyst/{pipeline => }/model/training/models/__init__.py (100%) rename epochalyst/{pipeline => }/model/training/models/timm.py (100%) rename epochalyst/{pipeline => }/model/training/pretrain_block.py (97%) rename epochalyst/{pipeline => }/model/training/torch_trainer.py (99%) rename epochalyst/{pipeline => }/model/training/training.py (88%) rename epochalyst/{pipeline => }/model/training/training_block.py (97%) rename epochalyst/{pipeline => }/model/training/utils/__init__.py (100%) rename epochalyst/{pipeline => }/model/training/utils/get_dependencies.py (100%) rename epochalyst/{pipeline => }/model/training/utils/recursive_repr.py (100%) rename epochalyst/{pipeline => }/model/training/utils/tensor_functions.py (100%) rename epochalyst/{pipeline => }/model/transformation/__init__.py (100%) rename epochalyst/{pipeline => }/model/transformation/transformation.py (91%) rename epochalyst/{pipeline => }/model/transformation/transformation_block.py (94%) delete mode 100644 epochalyst/pipeline/__init__.py create mode 100644 epochalyst/utils/__init__.py rename epochalyst/{_core/_caching/_cacher.py => utils/cacher.py} (97%) rename epochalyst/{logging => utils}/logger.py (97%) create mode 100644 tests/model/__init__.py rename tests/{pipeline => model}/test_ensemble.py (92%) rename tests/{pipeline => }/model/test_model.py (91%) rename tests/{pipeline => model}/test_recursive_repr.py (89%) create mode 100644 tests/model/training/__init__.py create mode 100644 tests/model/training/augmentation/__init__.py rename tests/{pipeline => }/model/training/augmentation/test_audio/white_noise.wav (100%) rename tests/{pipeline => }/model/training/augmentation/test_image_augmentations.py (96%) rename tests/{pipeline => }/model/training/augmentation/test_time_series_augmentations.py (98%) rename tests/{pipeline => }/model/training/augmentation/test_utils.py (97%) create mode 100644 tests/model/training/models/__init__.py rename tests/{pipeline => }/model/training/models/test_timm.py (81%) rename tests/{pipeline => }/model/training/test_pretrain_block.py (89%) rename tests/{pipeline => }/model/training/test_torch_trainer.py (98%) rename tests/{pipeline => }/model/training/test_training.py (96%) rename tests/{pipeline => }/model/training/test_training_block.py (95%) create mode 100644 tests/model/training/utils/__init__.py rename tests/{pipeline => }/model/training/utils/test_get_dependencies.py (95%) create mode 100644 tests/model/transformation/__init__.py rename tests/{pipeline => }/model/transformation/test_transformation.py (95%) rename tests/{pipeline => }/model/transformation/test_transformation_block.py (96%) create mode 100644 tests/utils/__init__.py rename tests/{_core/_caching => utils}/test__cacher.py (99%) rename tests/{logging => utils}/test_logger.py (97%) diff --git a/epochalyst/_core/__init__.py b/epochalyst/_core/__init__.py deleted file mode 100644 index 3835ea8..0000000 --- a/epochalyst/_core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Module containing the core functionality of the epochalyst package.""" diff --git a/epochalyst/_core/_caching/__init__.py b/epochalyst/_core/_caching/__init__.py deleted file mode 100644 index d1647e3..0000000 --- a/epochalyst/_core/_caching/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Module for core caching functionality.""" diff --git a/epochalyst/_core/_pipeline/__init__.py b/epochalyst/_core/_pipeline/__init__.py deleted file mode 100644 index 1393da4..0000000 --- a/epochalyst/_core/_pipeline/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Module for core pipeline functionality.""" diff --git a/epochalyst/logging/__init__.py b/epochalyst/logging/__init__.py deleted file mode 100644 index 9207955..0000000 --- a/epochalyst/logging/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Module for core logging functionality.""" diff --git a/epochalyst/pipeline/model/__init__.py b/epochalyst/model/__init__.py similarity index 100% rename from epochalyst/pipeline/model/__init__.py rename to epochalyst/model/__init__.py diff --git a/epochalyst/_core/_pipeline/_custom_data_parallel.py b/epochalyst/model/_custom_data_parallel.py similarity index 100% rename from epochalyst/_core/_pipeline/_custom_data_parallel.py rename to epochalyst/model/_custom_data_parallel.py diff --git a/epochalyst/pipeline/ensemble.py b/epochalyst/model/ensemble.py similarity index 96% rename from epochalyst/pipeline/ensemble.py rename to epochalyst/model/ensemble.py index 9379e7f..774dc38 100644 --- a/epochalyst/pipeline/ensemble.py +++ b/epochalyst/model/ensemble.py @@ -4,7 +4,7 @@ from agogos.training import ParallelTrainingSystem -from epochalyst._core._caching._cacher import CacheArgs +from epochalyst.utils.cacher import CacheArgs class EnsemblePipeline(ParallelTrainingSystem): diff --git a/epochalyst/pipeline/model/model.py b/epochalyst/model/model.py similarity index 92% rename from epochalyst/pipeline/model/model.py rename to epochalyst/model/model.py index 500f19f..b54c119 100644 --- a/epochalyst/pipeline/model/model.py +++ b/epochalyst/model/model.py @@ -1,11 +1,8 @@ -"""ModelPipeline connects multiple transforming and training systems for extended training functionality.""" - from typing import Any +from epochalyst.utils.cacher import CacheArgs from agogos.training import Pipeline -from epochalyst._core._caching._cacher import CacheArgs - class ModelPipeline(Pipeline): """ModelPipeline is the class used to create the pipeline for the model. (Currently same implementation as agogos pipeline). diff --git a/epochalyst/pipeline/model/training/__init__.py b/epochalyst/model/training/__init__.py similarity index 100% rename from epochalyst/pipeline/model/training/__init__.py rename to epochalyst/model/training/__init__.py diff --git a/epochalyst/pipeline/model/training/augmentation/__init__.py b/epochalyst/model/training/augmentation/__init__.py similarity index 100% rename from epochalyst/pipeline/model/training/augmentation/__init__.py rename to epochalyst/model/training/augmentation/__init__.py diff --git a/epochalyst/pipeline/model/training/augmentation/image_augmentations.py b/epochalyst/model/training/augmentation/image_augmentations.py similarity index 100% rename from epochalyst/pipeline/model/training/augmentation/image_augmentations.py rename to epochalyst/model/training/augmentation/image_augmentations.py diff --git a/epochalyst/pipeline/model/training/augmentation/time_series_augmentations.py b/epochalyst/model/training/augmentation/time_series_augmentations.py similarity index 99% rename from epochalyst/pipeline/model/training/augmentation/time_series_augmentations.py rename to epochalyst/model/training/augmentation/time_series_augmentations.py index 35def54..c94853a 100644 --- a/epochalyst/pipeline/model/training/augmentation/time_series_augmentations.py +++ b/epochalyst/model/training/augmentation/time_series_augmentations.py @@ -6,7 +6,7 @@ import numpy as np import torch -from epochalyst.pipeline.model.training.augmentation.utils import get_audiomentations +from epochalyst.model.training.augmentation.utils import get_audiomentations @dataclass diff --git a/epochalyst/pipeline/model/training/augmentation/utils.py b/epochalyst/model/training/augmentation/utils.py similarity index 98% rename from epochalyst/pipeline/model/training/augmentation/utils.py rename to epochalyst/model/training/augmentation/utils.py index 3addfe9..581702e 100644 --- a/epochalyst/pipeline/model/training/augmentation/utils.py +++ b/epochalyst/model/training/augmentation/utils.py @@ -12,7 +12,7 @@ import torch -from epochalyst.pipeline.model.training.utils.recursive_repr import recursive_repr +from epochalyst.model.training.utils.recursive_repr import recursive_repr def get_audiomentations() -> ModuleType: diff --git a/epochalyst/pipeline/model/training/models/__init__.py b/epochalyst/model/training/models/__init__.py similarity index 100% rename from epochalyst/pipeline/model/training/models/__init__.py rename to epochalyst/model/training/models/__init__.py diff --git a/epochalyst/pipeline/model/training/models/timm.py b/epochalyst/model/training/models/timm.py similarity index 100% rename from epochalyst/pipeline/model/training/models/timm.py rename to epochalyst/model/training/models/timm.py diff --git a/epochalyst/pipeline/model/training/pretrain_block.py b/epochalyst/model/training/pretrain_block.py similarity index 97% rename from epochalyst/pipeline/model/training/pretrain_block.py rename to epochalyst/model/training/pretrain_block.py index cab6ed5..361a9d5 100644 --- a/epochalyst/pipeline/model/training/pretrain_block.py +++ b/epochalyst/model/training/pretrain_block.py @@ -6,7 +6,7 @@ from joblib import hash -from epochalyst.pipeline.model.training.training_block import TrainingBlock +from epochalyst.model.training.training_block import TrainingBlock @dataclass diff --git a/epochalyst/pipeline/model/training/torch_trainer.py b/epochalyst/model/training/torch_trainer.py similarity index 99% rename from epochalyst/pipeline/model/training/torch_trainer.py rename to epochalyst/model/training/torch_trainer.py index 35cd4b6..87df3ba 100644 --- a/epochalyst/pipeline/model/training/torch_trainer.py +++ b/epochalyst/model/training/torch_trainer.py @@ -19,10 +19,10 @@ from torch.utils.data import DataLoader, Dataset, TensorDataset from tqdm import tqdm -from epochalyst._core._pipeline._custom_data_parallel import _CustomDataParallel -from epochalyst.pipeline.model.training.training_block import TrainingBlock -from epochalyst.pipeline.model.training.utils import _get_onnxrt, _get_openvino -from epochalyst.pipeline.model.training.utils.tensor_functions import batch_to_device +from epochalyst.model._custom_data_parallel import _CustomDataParallel +from epochalyst.model.training.training_block import TrainingBlock +from epochalyst.model.training.utils import _get_onnxrt, _get_openvino +from epochalyst.model.training.utils.tensor_functions import batch_to_device T = TypeVar("T", bound=Dataset) # type: ignore[type-arg] T_co = TypeVar("T_co", covariant=True) diff --git a/epochalyst/pipeline/model/training/training.py b/epochalyst/model/training/training.py similarity index 88% rename from epochalyst/pipeline/model/training/training.py rename to epochalyst/model/training/training.py index 1017c07..8becc5b 100644 --- a/epochalyst/pipeline/model/training/training.py +++ b/epochalyst/model/training/training.py @@ -4,10 +4,10 @@ from agogos.training import TrainingSystem, TrainType -from epochalyst._core._caching._cacher import CacheArgs, _Cacher +from epochalyst.utils.cacher import CacheArgs, Cacher -class TrainingPipeline(TrainingSystem, _Cacher): +class TrainingPipeline(TrainingSystem, Cacher): """The training pipeline. This is the class used to create the pipeline for the training of the model. :param steps: The steps to train the model. @@ -35,9 +35,9 @@ def train(self, x: Any, y: Any, cache_args: CacheArgs | None = None, **train_arg # Furthest step for i, step in enumerate(self.get_steps()): - # Check if step is instance of _Cacher and if cache_args exists - if not isinstance(step, _Cacher) or not isinstance(step, TrainType): - self.log_to_debug(f"{step} is not instance of _Cacher or TrainType") + # Check if step is instance of Cacher and if cache_args exists + if not isinstance(step, Cacher) or not isinstance(step, TrainType): + self.log_to_debug(f"{step} is not instance of Cacher or TrainType") continue step_args = train_args.get(step.__class__.__name__, None) @@ -89,9 +89,9 @@ def predict(self, x: Any, cache_args: CacheArgs | None = None, **pred_args: Any) # Retrieve furthest step calculated for i, step in enumerate(self.get_steps()): - # Check if step is instance of _Cacher and if cache_args exists - if not isinstance(step, _Cacher) or not isinstance(step, TrainType): - self.log_to_debug(f"{step} is not instance of _Cacher or TrainType") + # Check if step is instance of Cacher and if cache_args exists + if not isinstance(step, Cacher) or not isinstance(step, TrainType): + self.log_to_debug(f"{step} is not instance of Cacher or TrainType") continue step_args = pred_args.get(step.__class__.__name__, None) diff --git a/epochalyst/pipeline/model/training/training_block.py b/epochalyst/model/training/training_block.py similarity index 97% rename from epochalyst/pipeline/model/training/training_block.py rename to epochalyst/model/training/training_block.py index 3f35d73..fa18fe7 100644 --- a/epochalyst/pipeline/model/training/training_block.py +++ b/epochalyst/model/training/training_block.py @@ -5,10 +5,10 @@ from agogos.training import Trainer -from epochalyst._core._caching._cacher import CacheArgs, _Cacher +from epochalyst.utils.cacher import CacheArgs, Cacher -class TrainingBlock(Trainer, _Cacher): +class TrainingBlock(Trainer, Cacher): """The training block is a flexible block that allows for training of any model. Methods diff --git a/epochalyst/pipeline/model/training/utils/__init__.py b/epochalyst/model/training/utils/__init__.py similarity index 100% rename from epochalyst/pipeline/model/training/utils/__init__.py rename to epochalyst/model/training/utils/__init__.py diff --git a/epochalyst/pipeline/model/training/utils/get_dependencies.py b/epochalyst/model/training/utils/get_dependencies.py similarity index 100% rename from epochalyst/pipeline/model/training/utils/get_dependencies.py rename to epochalyst/model/training/utils/get_dependencies.py diff --git a/epochalyst/pipeline/model/training/utils/recursive_repr.py b/epochalyst/model/training/utils/recursive_repr.py similarity index 100% rename from epochalyst/pipeline/model/training/utils/recursive_repr.py rename to epochalyst/model/training/utils/recursive_repr.py diff --git a/epochalyst/pipeline/model/training/utils/tensor_functions.py b/epochalyst/model/training/utils/tensor_functions.py similarity index 100% rename from epochalyst/pipeline/model/training/utils/tensor_functions.py rename to epochalyst/model/training/utils/tensor_functions.py diff --git a/epochalyst/pipeline/model/transformation/__init__.py b/epochalyst/model/transformation/__init__.py similarity index 100% rename from epochalyst/pipeline/model/transformation/__init__.py rename to epochalyst/model/transformation/__init__.py diff --git a/epochalyst/pipeline/model/transformation/transformation.py b/epochalyst/model/transformation/transformation.py similarity index 91% rename from epochalyst/pipeline/model/transformation/transformation.py rename to epochalyst/model/transformation/transformation.py index 463dc0c..854b7c4 100644 --- a/epochalyst/pipeline/model/transformation/transformation.py +++ b/epochalyst/model/transformation/transformation.py @@ -1,15 +1,14 @@ -"""TransformationPipeline that extends from TransformingSystem, _Cacher and _Logger.""" - +"""TransformationPipeline that extends from TransformingSystem, Cacher and _Logger.""" from dataclasses import dataclass from typing import Any from agogos.transforming import TransformingSystem, TransformType -from epochalyst._core._caching._cacher import CacheArgs, _Cacher +from epochalyst.utils.cacher import Cacher, CacheArgs @dataclass -class TransformationPipeline(TransformingSystem, _Cacher): +class TransformationPipeline(TransformingSystem, Cacher): """TransformationPipeline is the class used to create the pipeline for the transformation of the data. ### Parameters: @@ -80,9 +79,9 @@ def transform(self, data: Any, cache_args: CacheArgs | None = None, **transform_ # Furthest step for i, step in enumerate(self.get_steps()): - # Check if step is instance of _Cacher and if cache_args exists - if not isinstance(step, _Cacher) or not isinstance(step, TransformType): - self.log_to_debug(f"{step} is not instance of _Cacher or TransformType") + # Check if step is instance of Cacher and if cache_args exists + if not isinstance(step, Cacher) or not isinstance(step, TransformType): + self.log_to_debug(f"{step} is not instance of Cacher or TransformType") continue step_args = transform_args.get(step.__class__.__name__, None) diff --git a/epochalyst/pipeline/model/transformation/transformation_block.py b/epochalyst/model/transformation/transformation_block.py similarity index 94% rename from epochalyst/pipeline/model/transformation/transformation_block.py rename to epochalyst/model/transformation/transformation_block.py index 8836ffa..8e2a834 100644 --- a/epochalyst/pipeline/model/transformation/transformation_block.py +++ b/epochalyst/model/transformation/transformation_block.py @@ -5,10 +5,10 @@ from agogos.transforming import Transformer -from epochalyst._core._caching._cacher import CacheArgs, _Cacher +from epochalyst.utils.cacher import Cacher, CacheArgs -class TransformationBlock(Transformer, _Cacher): +class TransformationBlock(Transformer, Cacher): """The transformation block is a flexible block that allows for transformation of any data. Methods @@ -63,8 +63,8 @@ def transform(self, data: Any, cache_args: CacheArgs | None = None, **transform_ :return: The transformed data. """ if cache_args and self.cache_exists( - name=self.get_hash(), - cache_args=cache_args, + name=self.get_hash(), + cache_args=cache_args, ): self.log_to_terminal( f"Cache exists for {self.__class__} with hash: {self.get_hash()}. Using the cache.", diff --git a/epochalyst/pipeline/__init__.py b/epochalyst/pipeline/__init__.py deleted file mode 100644 index 1e2a1f4..0000000 --- a/epochalyst/pipeline/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Module for all the data pipelines.""" diff --git a/epochalyst/utils/__init__.py b/epochalyst/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/epochalyst/_core/_caching/_cacher.py b/epochalyst/utils/cacher.py similarity index 97% rename from epochalyst/_core/_caching/_cacher.py rename to epochalyst/utils/cacher.py index 15d061e..00e7c52 100644 --- a/epochalyst/_core/_caching/_cacher.py +++ b/epochalyst/utils/cacher.py @@ -2,9 +2,10 @@ import os import pickle import sys -from typing import Any, Literal, TypedDict +from typing import TypedDict, Literal, Any +from typing_extensions import NotRequired -from epochalyst.logging.logger import Logger +from epochalyst.utils.logger import Logger try: import dask.array as da @@ -27,11 +28,10 @@ except ImportError: """User doesn't require these packages""" - if sys.version_info < (3, 11): # pragma: no cover ( bool: path_exists = os.path.exists(storage_path + name + ".parquet") elif storage_type == ".csv": # Check if the file exists or if there are any parts inside the folder - path_exists = os.path.exists(storage_path + name + ".csv") or glob.glob(storage_path + name + "/*.part") != [] + path_exists = os.path.exists(storage_path + name + ".csv") or glob.glob( + storage_path + name + "/*.part") != [] elif storage_type == ".npy_stack": path_exists = os.path.exists(storage_path + name) elif storage_type == ".pkl": @@ -245,7 +246,8 @@ def _get_cache(self, name: str, cache_args: CacheArgs | None = None) -> Any: # "storage_type must be .npy, .parquet, .csv, or .npy_stack, other types not supported yet", ) - def _store_cache(self, name: str, data: Any, cache_args: CacheArgs | None = None) -> None: # noqa: ANN401 C901 PLR0915 PLR0912 + def _store_cache(self, name: str, data: Any, + cache_args: CacheArgs | None = None) -> None: # noqa: ANN401 C901 PLR0915 PLR0912 """Store one set of data. :param name: The name of the cache. diff --git a/epochalyst/logging/logger.py b/epochalyst/utils/logger.py similarity index 97% rename from epochalyst/logging/logger.py rename to epochalyst/utils/logger.py index 935d175..03ff8a0 100644 --- a/epochalyst/logging/logger.py +++ b/epochalyst/utils/logger.py @@ -1,9 +1,6 @@ -"""Logger base class.""" - import logging import os -from collections.abc import Mapping -from typing import Any +from typing import Mapping, Any class Logger: diff --git a/tests/model/__init__.py b/tests/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pipeline/test_ensemble.py b/tests/model/test_ensemble.py similarity index 92% rename from tests/pipeline/test_ensemble.py rename to tests/model/test_ensemble.py index 42c7b55..1f0f80b 100644 --- a/tests/pipeline/test_ensemble.py +++ b/tests/model/test_ensemble.py @@ -5,12 +5,10 @@ import numpy as np import pytest -from epochalyst.pipeline.ensemble import EnsemblePipeline -from epochalyst.pipeline.model.model import ModelPipeline -from epochalyst.pipeline.model.transformation.transformation import \ - TransformationPipeline -from epochalyst.pipeline.model.transformation.transformation_block import \ - TransformationBlock +from epochalyst.model.ensemble import EnsemblePipeline +from epochalyst.model.model import ModelPipeline +from epochalyst.model.transformation.transformation import TransformationPipeline +from epochalyst.model.transformation.transformation_block import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/pipeline/model/test_model.py b/tests/model/test_model.py similarity index 91% rename from tests/pipeline/model/test_model.py rename to tests/model/test_model.py index 6c46b47..8ca7201 100644 --- a/tests/pipeline/model/test_model.py +++ b/tests/model/test_model.py @@ -5,11 +5,9 @@ import numpy as np import pytest -from epochalyst.pipeline.model.model import ModelPipeline -from epochalyst.pipeline.model.transformation.transformation import \ - TransformationPipeline -from epochalyst.pipeline.model.transformation.transformation_block import \ - TransformationBlock +from epochalyst.model.model import ModelPipeline +from epochalyst.model.transformation.transformation import TransformationPipeline +from epochalyst.model.transformation.transformation_block import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/pipeline/test_recursive_repr.py b/tests/model/test_recursive_repr.py similarity index 89% rename from tests/pipeline/test_recursive_repr.py rename to tests/model/test_recursive_repr.py index d98a93d..7e6f6b7 100644 --- a/tests/pipeline/test_recursive_repr.py +++ b/tests/model/test_recursive_repr.py @@ -1,6 +1,4 @@ -import pytest - -from epochalyst.pipeline.model.training.utils.recursive_repr import recursive_repr +from epochalyst.model.training.utils.recursive_repr import recursive_repr class TestRecursiveRepr: @@ -40,4 +38,4 @@ def __init__(self): def test_recursive_repr_with_no_dict(self): obj = 123 expected_repr = repr(obj) - assert recursive_repr(obj) == expected_repr \ No newline at end of file + assert recursive_repr(obj) == expected_repr diff --git a/tests/model/training/__init__.py b/tests/model/training/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/model/training/augmentation/__init__.py b/tests/model/training/augmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pipeline/model/training/augmentation/test_audio/white_noise.wav b/tests/model/training/augmentation/test_audio/white_noise.wav similarity index 100% rename from tests/pipeline/model/training/augmentation/test_audio/white_noise.wav rename to tests/model/training/augmentation/test_audio/white_noise.wav diff --git a/tests/pipeline/model/training/augmentation/test_image_augmentations.py b/tests/model/training/augmentation/test_image_augmentations.py similarity index 96% rename from tests/pipeline/model/training/augmentation/test_image_augmentations.py rename to tests/model/training/augmentation/test_image_augmentations.py index 323b880..fabb186 100644 --- a/tests/pipeline/model/training/augmentation/test_image_augmentations.py +++ b/tests/model/training/augmentation/test_image_augmentations.py @@ -1,6 +1,6 @@ import torch -from epochalyst.pipeline.model.training.augmentation import image_augmentations +from epochalyst.model.training.augmentation import image_augmentations class TestImageAugmentations: diff --git a/tests/pipeline/model/training/augmentation/test_time_series_augmentations.py b/tests/model/training/augmentation/test_time_series_augmentations.py similarity index 98% rename from tests/pipeline/model/training/augmentation/test_time_series_augmentations.py rename to tests/model/training/augmentation/test_time_series_augmentations.py index 02faca6..4f7e102 100644 --- a/tests/pipeline/model/training/augmentation/test_time_series_augmentations.py +++ b/tests/model/training/augmentation/test_time_series_augmentations.py @@ -1,7 +1,7 @@ import numpy as np import torch -from epochalyst.pipeline.model.training.augmentation import \ +from epochalyst.model.training.augmentation import \ time_series_augmentations @@ -212,4 +212,4 @@ def test_add_background_noise_wrapper(self): augmented_x = add_background_noise_wrapper(x, sr) # verify that not applying augmentation doesnt do anything - assert not torch.all(x == augmented_x) \ No newline at end of file + assert not torch.all(x == augmented_x) diff --git a/tests/pipeline/model/training/augmentation/test_utils.py b/tests/model/training/augmentation/test_utils.py similarity index 97% rename from tests/pipeline/model/training/augmentation/test_utils.py rename to tests/model/training/augmentation/test_utils.py index df4aabc..5f101f3 100644 --- a/tests/pipeline/model/training/augmentation/test_utils.py +++ b/tests/model/training/augmentation/test_utils.py @@ -1,6 +1,6 @@ import torch -from epochalyst.pipeline.model.training.augmentation import utils +from epochalyst.model.training.augmentation import utils def set_torch_seed(seed: int = 42) -> None: diff --git a/tests/model/training/models/__init__.py b/tests/model/training/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pipeline/model/training/models/test_timm.py b/tests/model/training/models/test_timm.py similarity index 81% rename from tests/pipeline/model/training/models/test_timm.py rename to tests/model/training/models/test_timm.py index 65b7aac..d4fe585 100644 --- a/tests/pipeline/model/training/models/test_timm.py +++ b/tests/model/training/models/test_timm.py @@ -1,6 +1,6 @@ import torch -from epochalyst.pipeline.model.training.models.timm import Timm +from epochalyst.model.training.models.timm import Timm class TestTimm: diff --git a/tests/pipeline/model/training/test_pretrain_block.py b/tests/model/training/test_pretrain_block.py similarity index 89% rename from tests/pipeline/model/training/test_pretrain_block.py rename to tests/model/training/test_pretrain_block.py index 9c4cdd4..5d554eb 100644 --- a/tests/pipeline/model/training/test_pretrain_block.py +++ b/tests/model/training/test_pretrain_block.py @@ -1,6 +1,6 @@ import pytest -from epochalyst.pipeline.model.training.pretrain_block import PretrainBlock +from epochalyst.model.training.pretrain_block import PretrainBlock class TestPretrainBlock: diff --git a/tests/pipeline/model/training/test_torch_trainer.py b/tests/model/training/test_torch_trainer.py similarity index 98% rename from tests/pipeline/model/training/test_torch_trainer.py rename to tests/model/training/test_torch_trainer.py index 0d55800..5166099 100644 --- a/tests/pipeline/model/training/test_torch_trainer.py +++ b/tests/model/training/test_torch_trainer.py @@ -4,11 +4,9 @@ import time from dataclasses import dataclass -from epochalyst._core._pipeline._custom_data_parallel import _CustomDataParallel -from epochalyst.pipeline.model.training.utils import _get_onnxrt, _get_openvino -from types import ModuleType +from epochalyst.model._custom_data_parallel import _CustomDataParallel -from epochalyst.pipeline.model.training.torch_trainer import custom_collate +from epochalyst.model.training.torch_trainer import custom_collate from typing import Any from unittest.mock import patch @@ -16,7 +14,7 @@ import pytest import torch -from epochalyst.pipeline.model.training.torch_trainer import TorchTrainer +from epochalyst.model.training.torch_trainer import TorchTrainer from tests.constants import TEMP_DIR diff --git a/tests/pipeline/model/training/test_training.py b/tests/model/training/test_training.py similarity index 96% rename from tests/pipeline/model/training/test_training.py rename to tests/model/training/test_training.py index 3891b2a..8d1389b 100644 --- a/tests/pipeline/model/training/test_training.py +++ b/tests/model/training/test_training.py @@ -1,11 +1,9 @@ -from pathlib import Path - import numpy as np import pytest from agogos.training import Trainer -from epochalyst.pipeline.model.training.training import TrainingPipeline -from epochalyst.pipeline.model.training.training_block import TrainingBlock +from epochalyst.model.training.training import TrainingPipeline +from epochalyst.model.training.training_block import TrainingBlock from tests.constants import TEMP_DIR diff --git a/tests/pipeline/model/training/test_training_block.py b/tests/model/training/test_training_block.py similarity index 95% rename from tests/pipeline/model/training/test_training_block.py rename to tests/model/training/test_training_block.py index 15d7a35..4e4343c 100644 --- a/tests/pipeline/model/training/test_training_block.py +++ b/tests/model/training/test_training_block.py @@ -1,9 +1,8 @@ -import shutil from pathlib import Path import pytest -from epochalyst.pipeline.model.training.training_block import TrainingBlock +from epochalyst.model.training.training_block import TrainingBlock TEMP_DIR = Path("tests/temp") diff --git a/tests/model/training/utils/__init__.py b/tests/model/training/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pipeline/model/training/utils/test_get_dependencies.py b/tests/model/training/utils/test_get_dependencies.py similarity index 95% rename from tests/pipeline/model/training/utils/test_get_dependencies.py rename to tests/model/training/utils/test_get_dependencies.py index e1938ac..70823a6 100644 --- a/tests/pipeline/model/training/utils/test_get_dependencies.py +++ b/tests/model/training/utils/test_get_dependencies.py @@ -3,7 +3,7 @@ import pytest -from epochalyst.pipeline.model.training.utils import _get_openvino, _get_onnxrt +from epochalyst.model.training.utils import _get_openvino, _get_onnxrt class TestGetDependencies: diff --git a/tests/model/transformation/__init__.py b/tests/model/transformation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pipeline/model/transformation/test_transformation.py b/tests/model/transformation/test_transformation.py similarity index 95% rename from tests/pipeline/model/transformation/test_transformation.py rename to tests/model/transformation/test_transformation.py index e9444f8..afdbf9b 100644 --- a/tests/pipeline/model/transformation/test_transformation.py +++ b/tests/model/transformation/test_transformation.py @@ -5,10 +5,8 @@ import pytest from agogos.transforming import Transformer -from epochalyst.pipeline.model.transformation.transformation import \ - TransformationPipeline -from epochalyst.pipeline.model.transformation.transformation_block import \ - TransformationBlock +from epochalyst.model.transformation.transformation import TransformationPipeline +from epochalyst.model.transformation.transformation_block import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/pipeline/model/transformation/test_transformation_block.py b/tests/model/transformation/test_transformation_block.py similarity index 96% rename from tests/pipeline/model/transformation/test_transformation_block.py rename to tests/model/transformation/test_transformation_block.py index b62aa8f..ee1df73 100644 --- a/tests/pipeline/model/transformation/test_transformation_block.py +++ b/tests/model/transformation/test_transformation_block.py @@ -6,8 +6,7 @@ import numpy as np import pytest -from epochalyst.pipeline.model.transformation.transformation_block import \ - TransformationBlock +from epochalyst.model.transformation.transformation_block import TransformationBlock TEMP_DIR = Path("tests/temp") diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/_core/_caching/test__cacher.py b/tests/utils/test__cacher.py similarity index 99% rename from tests/_core/_caching/test__cacher.py rename to tests/utils/test__cacher.py index 042e398..125158e 100644 --- a/tests/_core/_caching/test__cacher.py +++ b/tests/utils/test__cacher.py @@ -5,12 +5,12 @@ import polars as pl import pytest -from epochalyst._core._caching._cacher import _Cacher -from epochalyst.logging.logger import Logger +from epochalyst.utils.cacher import Cacher +from epochalyst.utils.logger import Logger from tests.constants import TEMP_DIR -class Implemented_Cacher(_Cacher, Logger): +class Implemented_Cacher(Cacher, Logger): pass diff --git a/tests/logging/test_logger.py b/tests/utils/test_logger.py similarity index 97% rename from tests/logging/test_logger.py rename to tests/utils/test_logger.py index cfb8ca5..0034ee8 100644 --- a/tests/logging/test_logger.py +++ b/tests/utils/test_logger.py @@ -3,7 +3,7 @@ import pytest -from epochalyst.logging.logger import Logger +from epochalyst.utils.logger import Logger test_string = "Test" From 0a7627a8269bd6164bcb10ec27424120c58be652 Mon Sep 17 00:00:00 2001 From: ksandor Date: Thu, 27 Jun 2024 16:52:41 +0200 Subject: [PATCH 2/7] 162 | shortened imports --- epochalyst/model/__init__.py | 33 +++++++++++++++++++ epochalyst/model/ensemble.py | 2 +- epochalyst/model/model.py | 5 ++- epochalyst/model/training/__init__.py | 13 ++++++++ .../{ => training}/_custom_data_parallel.py | 0 .../model/training/augmentation/__init__.py | 8 +++++ .../augmentation/time_series_augmentations.py | 2 +- epochalyst/model/training/models/__init__.py | 6 ++++ epochalyst/model/training/pretrain_block.py | 2 +- epochalyst/model/training/torch_trainer.py | 9 +++-- epochalyst/model/training/training.py | 2 +- epochalyst/model/training/training_block.py | 2 +- epochalyst/model/training/utils/__init__.py | 4 +++ epochalyst/model/transformation/__init__.py | 8 +++++ .../model/transformation/transformation.py | 3 +- .../transformation/transformation_block.py | 6 ++-- epochalyst/utils/__init__.py | 6 ++++ epochalyst/utils/cacher.py | 13 ++++---- epochalyst/utils/logger.py | 4 ++- tests/model/test_ensemble.py | 8 ++--- .../augmentation/test_image_augmentations.py | 2 +- .../test_time_series_augmentations.py | 3 +- .../model/training/augmentation/test_utils.py | 2 +- tests/model/training/models/test_timm.py | 2 +- tests/model/training/test_torch_trainer.py | 2 +- 25 files changed, 115 insertions(+), 32 deletions(-) rename epochalyst/model/{ => training}/_custom_data_parallel.py (100%) diff --git a/epochalyst/model/__init__.py b/epochalyst/model/__init__.py index cdaa5ca..b8ce5db 100644 --- a/epochalyst/model/__init__.py +++ b/epochalyst/model/__init__.py @@ -1 +1,34 @@ """Module containing model pipeline and child classes.""" + +# import from . +from epochalyst.model.ensemble import EnsemblePipeline +from epochalyst.model.model import ModelPipeline + +# import everything from submodules +from epochalyst.model.training import PretrainBlock, TorchTrainer, TrainingBlock, TrainingPipeline, TrainValidationDataset +from epochalyst.model.training._custom_data_parallel import _CustomDataParallel +from epochalyst.model.training.augmentation import image_augmentations, time_series_augmentations, utils +from epochalyst.model.training.models import Timm +from epochalyst.model.training.utils import _get_onnxrt, _get_openvino, batch_to_device, recursive_repr +from epochalyst.model.transformation import TransformationBlock, TransformationPipeline + +__all__ = [ + "EnsemblePipeline", + "ModelPipeline", + "PretrainBlock", + "TorchTrainer", + "TrainingBlock", + "TrainingPipeline", + "TrainValidationDataset", + "_CustomDataParallel", + "image_augmentations", + "time_series_augmentations", + "utils", + "Timm", + "_get_onnxrt", + "_get_openvino", + "batch_to_device", + "recursive_repr", + "TransformationPipeline", + "TransformationBlock", +] diff --git a/epochalyst/model/ensemble.py b/epochalyst/model/ensemble.py index 774dc38..370ee78 100644 --- a/epochalyst/model/ensemble.py +++ b/epochalyst/model/ensemble.py @@ -4,7 +4,7 @@ from agogos.training import ParallelTrainingSystem -from epochalyst.utils.cacher import CacheArgs +from epochalyst.utils import CacheArgs class EnsemblePipeline(ParallelTrainingSystem): diff --git a/epochalyst/model/model.py b/epochalyst/model/model.py index b54c119..3687e36 100644 --- a/epochalyst/model/model.py +++ b/epochalyst/model/model.py @@ -1,8 +1,11 @@ +"""Model module. Contains the ModelPipeline class.""" + from typing import Any -from epochalyst.utils.cacher import CacheArgs from agogos.training import Pipeline +from epochalyst.utils import CacheArgs + class ModelPipeline(Pipeline): """ModelPipeline is the class used to create the pipeline for the model. (Currently same implementation as agogos pipeline). diff --git a/epochalyst/model/training/__init__.py b/epochalyst/model/training/__init__.py index 58fca67..46e26fa 100644 --- a/epochalyst/model/training/__init__.py +++ b/epochalyst/model/training/__init__.py @@ -1 +1,14 @@ """Module containing training functionality for the epochalyst package.""" + +from .pretrain_block import PretrainBlock +from .torch_trainer import TorchTrainer, TrainValidationDataset +from .training import TrainingPipeline +from .training_block import TrainingBlock + +__all__ = [ + "PretrainBlock", + "TrainingBlock", + "TorchTrainer", + "TrainingPipeline", + "TrainValidationDataset", +] diff --git a/epochalyst/model/_custom_data_parallel.py b/epochalyst/model/training/_custom_data_parallel.py similarity index 100% rename from epochalyst/model/_custom_data_parallel.py rename to epochalyst/model/training/_custom_data_parallel.py diff --git a/epochalyst/model/training/augmentation/__init__.py b/epochalyst/model/training/augmentation/__init__.py index 758ab3e..336fc3d 100644 --- a/epochalyst/model/training/augmentation/__init__.py +++ b/epochalyst/model/training/augmentation/__init__.py @@ -1 +1,9 @@ """Module containing implementation for augmentations.""" + +from epochalyst.model.training.augmentation import image_augmentations, time_series_augmentations, utils + +__all__ = [ + "image_augmentations", + "time_series_augmentations", + "utils", +] diff --git a/epochalyst/model/training/augmentation/time_series_augmentations.py b/epochalyst/model/training/augmentation/time_series_augmentations.py index c94853a..8cf9759 100644 --- a/epochalyst/model/training/augmentation/time_series_augmentations.py +++ b/epochalyst/model/training/augmentation/time_series_augmentations.py @@ -6,7 +6,7 @@ import numpy as np import torch -from epochalyst.model.training.augmentation.utils import get_audiomentations +from .utils import get_audiomentations @dataclass diff --git a/epochalyst/model/training/models/__init__.py b/epochalyst/model/training/models/__init__.py index 7dcd068..165fb70 100644 --- a/epochalyst/model/training/models/__init__.py +++ b/epochalyst/model/training/models/__init__.py @@ -1 +1,7 @@ """Module for reusable models or wrappers.""" + +from .timm import Timm + +__all__ = [ + "Timm", +] diff --git a/epochalyst/model/training/pretrain_block.py b/epochalyst/model/training/pretrain_block.py index 361a9d5..7aaef74 100644 --- a/epochalyst/model/training/pretrain_block.py +++ b/epochalyst/model/training/pretrain_block.py @@ -6,7 +6,7 @@ from joblib import hash -from epochalyst.model.training.training_block import TrainingBlock +from .training_block import TrainingBlock @dataclass diff --git a/epochalyst/model/training/torch_trainer.py b/epochalyst/model/training/torch_trainer.py index 87df3ba..759d2d1 100644 --- a/epochalyst/model/training/torch_trainer.py +++ b/epochalyst/model/training/torch_trainer.py @@ -19,10 +19,9 @@ from torch.utils.data import DataLoader, Dataset, TensorDataset from tqdm import tqdm -from epochalyst.model._custom_data_parallel import _CustomDataParallel -from epochalyst.model.training.training_block import TrainingBlock -from epochalyst.model.training.utils import _get_onnxrt, _get_openvino -from epochalyst.model.training.utils.tensor_functions import batch_to_device +from ._custom_data_parallel import _CustomDataParallel +from .training_block import TrainingBlock +from .utils import _get_onnxrt, _get_openvino, batch_to_device T = TypeVar("T", bound=Dataset) # type: ignore[type-arg] T_co = TypeVar("T_co", covariant=True) @@ -579,7 +578,7 @@ def create_dataloaders( validation_dataset, batch_size=self.batch_size, shuffle=False, - collate_fn=(self.collate_fn if hasattr(validation_dataset, "__getitems__") else None), # type: ignore[arg-type] + collate_fn=(self.collate_fn if hasattr(validation_dataset, "__getitems__") else None), **self.dataloader_args, ) return train_loader, validation_loader diff --git a/epochalyst/model/training/training.py b/epochalyst/model/training/training.py index 8becc5b..e8c8471 100644 --- a/epochalyst/model/training/training.py +++ b/epochalyst/model/training/training.py @@ -4,7 +4,7 @@ from agogos.training import TrainingSystem, TrainType -from epochalyst.utils.cacher import CacheArgs, Cacher +from epochalyst.utils import CacheArgs, Cacher class TrainingPipeline(TrainingSystem, Cacher): diff --git a/epochalyst/model/training/training_block.py b/epochalyst/model/training/training_block.py index fa18fe7..d13089e 100644 --- a/epochalyst/model/training/training_block.py +++ b/epochalyst/model/training/training_block.py @@ -5,7 +5,7 @@ from agogos.training import Trainer -from epochalyst.utils.cacher import CacheArgs, Cacher +from epochalyst.utils import CacheArgs, Cacher class TrainingBlock(Trainer, Cacher): diff --git a/epochalyst/model/training/utils/__init__.py b/epochalyst/model/training/utils/__init__.py index 526b575..5b91f9e 100644 --- a/epochalyst/model/training/utils/__init__.py +++ b/epochalyst/model/training/utils/__init__.py @@ -1,8 +1,12 @@ """Module with utility functions for training.""" from .get_dependencies import _get_onnxrt, _get_openvino +from .recursive_repr import recursive_repr +from .tensor_functions import batch_to_device __all__ = [ "_get_onnxrt", "_get_openvino", + "batch_to_device", + "recursive_repr", ] diff --git a/epochalyst/model/transformation/__init__.py b/epochalyst/model/transformation/__init__.py index 580522d..3a85565 100644 --- a/epochalyst/model/transformation/__init__.py +++ b/epochalyst/model/transformation/__init__.py @@ -1 +1,9 @@ """Module containing transformation functions for the model pipeline.""" + +from .transformation import TransformationPipeline +from .transformation_block import TransformationBlock + +__all__ = [ + "TransformationPipeline", + "TransformationBlock", +] diff --git a/epochalyst/model/transformation/transformation.py b/epochalyst/model/transformation/transformation.py index 854b7c4..6bb7bda 100644 --- a/epochalyst/model/transformation/transformation.py +++ b/epochalyst/model/transformation/transformation.py @@ -1,10 +1,11 @@ """TransformationPipeline that extends from TransformingSystem, Cacher and _Logger.""" + from dataclasses import dataclass from typing import Any from agogos.transforming import TransformingSystem, TransformType -from epochalyst.utils.cacher import Cacher, CacheArgs +from epochalyst.utils.cacher import CacheArgs, Cacher @dataclass diff --git a/epochalyst/model/transformation/transformation_block.py b/epochalyst/model/transformation/transformation_block.py index 8e2a834..1d5c215 100644 --- a/epochalyst/model/transformation/transformation_block.py +++ b/epochalyst/model/transformation/transformation_block.py @@ -5,7 +5,7 @@ from agogos.transforming import Transformer -from epochalyst.utils.cacher import Cacher, CacheArgs +from epochalyst.utils.cacher import CacheArgs, Cacher class TransformationBlock(Transformer, Cacher): @@ -63,8 +63,8 @@ def transform(self, data: Any, cache_args: CacheArgs | None = None, **transform_ :return: The transformed data. """ if cache_args and self.cache_exists( - name=self.get_hash(), - cache_args=cache_args, + name=self.get_hash(), + cache_args=cache_args, ): self.log_to_terminal( f"Cache exists for {self.__class__} with hash: {self.get_hash()}. Using the cache.", diff --git a/epochalyst/utils/__init__.py b/epochalyst/utils/__init__.py index e69de29..8069675 100644 --- a/epochalyst/utils/__init__.py +++ b/epochalyst/utils/__init__.py @@ -0,0 +1,6 @@ +"""Utils module, contains the Cacher and Logger classes.""" + +from .cacher import CacheArgs, Cacher +from .logger import Logger + +__all__ = ["CacheArgs", "Cacher", "Logger"] diff --git a/epochalyst/utils/cacher.py b/epochalyst/utils/cacher.py index 00e7c52..2f4cd1b 100644 --- a/epochalyst/utils/cacher.py +++ b/epochalyst/utils/cacher.py @@ -1,11 +1,14 @@ +"""The cacher module contains the Cacher class.""" + import glob import os import pickle import sys -from typing import TypedDict, Literal, Any +from typing import Any, Literal, TypedDict + from typing_extensions import NotRequired -from epochalyst.utils.logger import Logger +from .logger import Logger try: import dask.array as da @@ -125,8 +128,7 @@ def cache_exists(self, name: str, cache_args: CacheArgs | None = None) -> bool: path_exists = os.path.exists(storage_path + name + ".parquet") elif storage_type == ".csv": # Check if the file exists or if there are any parts inside the folder - path_exists = os.path.exists(storage_path + name + ".csv") or glob.glob( - storage_path + name + "/*.part") != [] + path_exists = os.path.exists(storage_path + name + ".csv") or glob.glob(storage_path + name + "/*.part") != [] elif storage_type == ".npy_stack": path_exists = os.path.exists(storage_path + name) elif storage_type == ".pkl": @@ -246,8 +248,7 @@ def _get_cache(self, name: str, cache_args: CacheArgs | None = None) -> Any: # "storage_type must be .npy, .parquet, .csv, or .npy_stack, other types not supported yet", ) - def _store_cache(self, name: str, data: Any, - cache_args: CacheArgs | None = None) -> None: # noqa: ANN401 C901 PLR0915 PLR0912 + def _store_cache(self, name: str, data: Any, cache_args: CacheArgs | None = None) -> None: # noqa: ANN401 C901 PLR0915 PLR0912 """Store one set of data. :param name: The name of the cache. diff --git a/epochalyst/utils/logger.py b/epochalyst/utils/logger.py index 03ff8a0..8836e70 100644 --- a/epochalyst/utils/logger.py +++ b/epochalyst/utils/logger.py @@ -1,6 +1,8 @@ +"""Logger base class for logging methods.""" + import logging import os -from typing import Mapping, Any +from typing import Any, Mapping class Logger: diff --git a/tests/model/test_ensemble.py b/tests/model/test_ensemble.py index 1f0f80b..73243b9 100644 --- a/tests/model/test_ensemble.py +++ b/tests/model/test_ensemble.py @@ -5,10 +5,10 @@ import numpy as np import pytest -from epochalyst.model.ensemble import EnsemblePipeline -from epochalyst.model.model import ModelPipeline -from epochalyst.model.transformation.transformation import TransformationPipeline -from epochalyst.model.transformation.transformation_block import TransformationBlock +from epochalyst.model import EnsemblePipeline +from epochalyst.model import ModelPipeline +from epochalyst.model import TransformationPipeline +from epochalyst.model import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/model/training/augmentation/test_image_augmentations.py b/tests/model/training/augmentation/test_image_augmentations.py index fabb186..149dd22 100644 --- a/tests/model/training/augmentation/test_image_augmentations.py +++ b/tests/model/training/augmentation/test_image_augmentations.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model.training.augmentation import image_augmentations +from epochalyst.model import image_augmentations class TestImageAugmentations: diff --git a/tests/model/training/augmentation/test_time_series_augmentations.py b/tests/model/training/augmentation/test_time_series_augmentations.py index 4f7e102..144cf9c 100644 --- a/tests/model/training/augmentation/test_time_series_augmentations.py +++ b/tests/model/training/augmentation/test_time_series_augmentations.py @@ -1,8 +1,7 @@ import numpy as np import torch -from epochalyst.model.training.augmentation import \ - time_series_augmentations +from epochalyst.model import time_series_augmentations def set_torch_seed(seed: int = 42) -> None: diff --git a/tests/model/training/augmentation/test_utils.py b/tests/model/training/augmentation/test_utils.py index 5f101f3..be40858 100644 --- a/tests/model/training/augmentation/test_utils.py +++ b/tests/model/training/augmentation/test_utils.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model.training.augmentation import utils +from epochalyst.model import utils def set_torch_seed(seed: int = 42) -> None: diff --git a/tests/model/training/models/test_timm.py b/tests/model/training/models/test_timm.py index d4fe585..ffde7c0 100644 --- a/tests/model/training/models/test_timm.py +++ b/tests/model/training/models/test_timm.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model.training.models.timm import Timm +from epochalyst.model import Timm class TestTimm: diff --git a/tests/model/training/test_torch_trainer.py b/tests/model/training/test_torch_trainer.py index 5166099..589229c 100644 --- a/tests/model/training/test_torch_trainer.py +++ b/tests/model/training/test_torch_trainer.py @@ -4,7 +4,7 @@ import time from dataclasses import dataclass -from epochalyst.model._custom_data_parallel import _CustomDataParallel +from epochalyst.model.training._custom_data_parallel import _CustomDataParallel from epochalyst.model.training.torch_trainer import custom_collate from typing import Any From 914b2620db2b0c9fc6e2ea938e688b41800cb476 Mon Sep 17 00:00:00 2001 From: Jeffrey Lim Date: Fri, 28 Jun 2024 09:52:38 +0200 Subject: [PATCH 3/7] CI: Configure pre-commit.ci (#213) * Remove pre-commit from GH actions * Small change to trigger CI * Attempt fix MyPy CI * Attempt fix MyPy CI 2 * Attempt fix MyPi ci 3 * Attempt fix MyPi CI 4 * Attempt fix MyPy CI 5 * Attempt fix MYPy CI 6 * Attempt fix MyPy CI 7 * Use pre-commit action * Create seperate workflow for pre-commit --- .github/workflows/main-branch-testing.yml | 13 ------------ .github/workflows/static-analysis.yml | 22 ++++++++++++++++++++ .github/workflows/version-branch-testing.yml | 12 ----------- .pre-commit-config.yaml | 6 ++++-- 4 files changed, 26 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/workflows/main-branch-testing.yml b/.github/workflows/main-branch-testing.yml index d672a3c..f6fc0de 100644 --- a/.github/workflows/main-branch-testing.yml +++ b/.github/workflows/main-branch-testing.yml @@ -7,20 +7,7 @@ on: branches: [ "main" ] jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - with: - python-version: 3.11 - - name: Install pre-commit - run: pip install pre-commit - - name: Run pre-commit - run: pre-commit run --all-files - build: - runs-on: ubuntu-latest container: image: python:3.11-slim diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml new file mode 100644 index 0000000..e8f2247 --- /dev/null +++ b/.github/workflows/static-analysis.yml @@ -0,0 +1,22 @@ +name: Static Analysis + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main", "v*" ] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4.1.6 + with: + fetch-depth: 0 + - name: Set up Python 3.10.14 + uses: actions/setup-python@v5.1.0 + with: + python-version: 3.10.14 + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/version-branch-testing.yml b/.github/workflows/version-branch-testing.yml index e779d4e..264ac2a 100644 --- a/.github/workflows/version-branch-testing.yml +++ b/.github/workflows/version-branch-testing.yml @@ -5,18 +5,6 @@ on: branches: ["v*"] jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.6 - - uses: actions/setup-python@v5.1.0 - with: - python-version: 3.10.14 - - name: Install pre-commit - run: pip install pre-commit - - name: Run pre-commit - run: pre-commit run --all-files - pytest: runs-on: ubuntu-latest steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d58cd4..d2784b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,6 @@ exclude: ^(external/|venv/|.venv/|tests/|.cache) +ci: + skip: [mypy] repos: - repo: local # Remove this when a new version of pre-commit-hooks (>4.6.0) is released hooks: @@ -68,10 +70,10 @@ repos: hooks: - id: mypy additional_dependencies: + - "--extra-index-url=https://download.pytorch.org/whl/cpu" - numpy==1.26.4 - pandas-stubs>=2.2.2.240514 - - matplotlib==3.8.4 - - torch==2.3.1 + - torch==2.3.1+cpu - dask==2024.6.2 - typing_extensions==4.9.0 - annotated-types==0.7.0 From fed1b4781b695146db7a4afa80e22bcb7c756997 Mon Sep 17 00:00:00 2001 From: Emiel Witting Date: Fri, 28 Jun 2024 10:55:15 +0200 Subject: [PATCH 4/7] remove model subdir and change augmentation imports --- epochalyst/__init__.py | 11 ++++-- epochalyst/{model => }/ensemble.py | 0 epochalyst/{model => }/model.py | 0 epochalyst/model/__init__.py | 34 ------------------ .../model/training/augmentation/__init__.py | 9 ----- epochalyst/{model => }/training/__init__.py | 0 .../training/_custom_data_parallel.py | 0 epochalyst/training/augmentation/__init__.py | 26 ++++++++++++++ .../augmentation/image_augmentations.py | 0 .../augmentation/time_series_augmentations.py | 2 +- .../training/augmentation/utils.py | 2 +- .../{model => }/training/models/__init__.py | 0 .../{model => }/training/models/timm.py | 0 .../{model => }/training/pretrain_block.py | 0 .../{model => }/training/torch_trainer.py | 18 ++++------ epochalyst/{model => }/training/training.py | 0 .../{model => }/training/training_block.py | 0 .../{model => }/training/utils/__init__.py | 0 .../training/utils/get_dependencies.py | 0 .../training/utils/recursive_repr.py | 0 .../training/utils/tensor_functions.py | 0 .../{model => }/transformation/__init__.py | 0 .../transformation/transformation.py | 0 .../transformation/transformation_block.py | 0 tests/model/transformation/__init__.py | 0 tests/{model => }/test_ensemble.py | 8 ++--- tests/{model => }/test_model.py | 6 ++-- tests/{model => training}/__init__.py | 0 .../augmentation}/__init__.py | 0 .../augmentation/test_audio/white_noise.wav | Bin .../augmentation/test_image_augmentations.py | 2 +- .../test_time_series_augmentations.py | 8 ++--- .../training/augmentation/test_utils.py | 2 +- .../models}/__init__.py | 0 .../{model => }/training/models/test_timm.py | 2 +- .../training/test_pretrain_block.py | 2 +- .../training/test_torch_trainer.py | 6 ++-- tests/{model => }/training/test_training.py | 4 +-- .../training/test_training_block.py | 2 +- .../models => training/utils}/__init__.py | 0 .../training/utils/test_get_dependencies.py | 2 +- .../utils}/test_recursive_repr.py | 2 +- .../utils => transformation}/__init__.py | 0 .../transformation/test_transformation.py | 4 +-- .../test_transformation_block.py | 2 +- 45 files changed, 70 insertions(+), 84 deletions(-) rename epochalyst/{model => }/ensemble.py (100%) rename epochalyst/{model => }/model.py (100%) delete mode 100644 epochalyst/model/__init__.py delete mode 100644 epochalyst/model/training/augmentation/__init__.py rename epochalyst/{model => }/training/__init__.py (100%) rename epochalyst/{model => }/training/_custom_data_parallel.py (100%) create mode 100644 epochalyst/training/augmentation/__init__.py rename epochalyst/{model => }/training/augmentation/image_augmentations.py (100%) rename epochalyst/{model => }/training/augmentation/time_series_augmentations.py (99%) rename epochalyst/{model => }/training/augmentation/utils.py (98%) rename epochalyst/{model => }/training/models/__init__.py (100%) rename epochalyst/{model => }/training/models/timm.py (100%) rename epochalyst/{model => }/training/pretrain_block.py (100%) rename epochalyst/{model => }/training/torch_trainer.py (98%) rename epochalyst/{model => }/training/training.py (100%) rename epochalyst/{model => }/training/training_block.py (100%) rename epochalyst/{model => }/training/utils/__init__.py (100%) rename epochalyst/{model => }/training/utils/get_dependencies.py (100%) rename epochalyst/{model => }/training/utils/recursive_repr.py (100%) rename epochalyst/{model => }/training/utils/tensor_functions.py (100%) rename epochalyst/{model => }/transformation/__init__.py (100%) rename epochalyst/{model => }/transformation/transformation.py (100%) rename epochalyst/{model => }/transformation/transformation_block.py (100%) delete mode 100644 tests/model/transformation/__init__.py rename tests/{model => }/test_ensemble.py (94%) rename tests/{model => }/test_model.py (92%) rename tests/{model => training}/__init__.py (100%) rename tests/{model/training => training/augmentation}/__init__.py (100%) rename tests/{model => }/training/augmentation/test_audio/white_noise.wav (100%) rename tests/{model => }/training/augmentation/test_image_augmentations.py (96%) rename tests/{model => }/training/augmentation/test_time_series_augmentations.py (96%) rename tests/{model => }/training/augmentation/test_utils.py (98%) rename tests/{model/training/augmentation => training/models}/__init__.py (100%) rename tests/{model => }/training/models/test_timm.py (86%) rename tests/{model => }/training/test_pretrain_block.py (90%) rename tests/{model => }/training/test_torch_trainer.py (99%) rename tests/{model => }/training/test_training.py (97%) rename tests/{model => }/training/test_training_block.py (96%) rename tests/{model/training/models => training/utils}/__init__.py (100%) rename tests/{model => }/training/utils/test_get_dependencies.py (96%) rename tests/{model => training/utils}/test_recursive_repr.py (94%) rename tests/{model/training/utils => transformation}/__init__.py (100%) rename tests/{model => }/transformation/test_transformation.py (96%) rename tests/{model => }/transformation/test_transformation_block.py (97%) diff --git a/epochalyst/__init__.py b/epochalyst/__init__.py index b5395a7..8662c7c 100644 --- a/epochalyst/__init__.py +++ b/epochalyst/__init__.py @@ -1,6 +1,13 @@ """The epochalyst package. It consists of the following modules: -- `logging`: The logging module contains the classes and methods to log the pipeline. -- `pipeline`: The pipeline module contains the classes and methods to create a pipeline for the model. + """ + +from .ensemble import EnsemblePipeline +from .model import ModelPipeline + +__all__ = [ + "ModelPipeline", + "EnsemblePipeline", +] diff --git a/epochalyst/model/ensemble.py b/epochalyst/ensemble.py similarity index 100% rename from epochalyst/model/ensemble.py rename to epochalyst/ensemble.py diff --git a/epochalyst/model/model.py b/epochalyst/model.py similarity index 100% rename from epochalyst/model/model.py rename to epochalyst/model.py diff --git a/epochalyst/model/__init__.py b/epochalyst/model/__init__.py deleted file mode 100644 index b8ce5db..0000000 --- a/epochalyst/model/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Module containing model pipeline and child classes.""" - -# import from . -from epochalyst.model.ensemble import EnsemblePipeline -from epochalyst.model.model import ModelPipeline - -# import everything from submodules -from epochalyst.model.training import PretrainBlock, TorchTrainer, TrainingBlock, TrainingPipeline, TrainValidationDataset -from epochalyst.model.training._custom_data_parallel import _CustomDataParallel -from epochalyst.model.training.augmentation import image_augmentations, time_series_augmentations, utils -from epochalyst.model.training.models import Timm -from epochalyst.model.training.utils import _get_onnxrt, _get_openvino, batch_to_device, recursive_repr -from epochalyst.model.transformation import TransformationBlock, TransformationPipeline - -__all__ = [ - "EnsemblePipeline", - "ModelPipeline", - "PretrainBlock", - "TorchTrainer", - "TrainingBlock", - "TrainingPipeline", - "TrainValidationDataset", - "_CustomDataParallel", - "image_augmentations", - "time_series_augmentations", - "utils", - "Timm", - "_get_onnxrt", - "_get_openvino", - "batch_to_device", - "recursive_repr", - "TransformationPipeline", - "TransformationBlock", -] diff --git a/epochalyst/model/training/augmentation/__init__.py b/epochalyst/model/training/augmentation/__init__.py deleted file mode 100644 index 336fc3d..0000000 --- a/epochalyst/model/training/augmentation/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Module containing implementation for augmentations.""" - -from epochalyst.model.training.augmentation import image_augmentations, time_series_augmentations, utils - -__all__ = [ - "image_augmentations", - "time_series_augmentations", - "utils", -] diff --git a/epochalyst/model/training/__init__.py b/epochalyst/training/__init__.py similarity index 100% rename from epochalyst/model/training/__init__.py rename to epochalyst/training/__init__.py diff --git a/epochalyst/model/training/_custom_data_parallel.py b/epochalyst/training/_custom_data_parallel.py similarity index 100% rename from epochalyst/model/training/_custom_data_parallel.py rename to epochalyst/training/_custom_data_parallel.py diff --git a/epochalyst/training/augmentation/__init__.py b/epochalyst/training/augmentation/__init__.py new file mode 100644 index 0000000..700a6a0 --- /dev/null +++ b/epochalyst/training/augmentation/__init__.py @@ -0,0 +1,26 @@ +"""Module containing implementation for augmentations.""" + +from epochalyst.training.augmentation.image_augmentations import CutMix, MixUp +from epochalyst.training.augmentation.time_series_augmentations import ( + AddBackgroundNoiseWrapper, + CutMix1D, + EnergyCutmix, + Mirror1D, + MixUp1D, + RandomAmplitudeShift, + RandomPhaseShift, + SubtractChannels, +) + +__all__ = [ + "CutMix", + "MixUp", + "CutMix1D", + "MixUp1D", + "Mirror1D", + "EnergyCutmix", + "RandomPhaseShift", + "RandomAmplitudeShift", + "SubtractChannels", + "AddBackgroundNoiseWrapper", +] diff --git a/epochalyst/model/training/augmentation/image_augmentations.py b/epochalyst/training/augmentation/image_augmentations.py similarity index 100% rename from epochalyst/model/training/augmentation/image_augmentations.py rename to epochalyst/training/augmentation/image_augmentations.py diff --git a/epochalyst/model/training/augmentation/time_series_augmentations.py b/epochalyst/training/augmentation/time_series_augmentations.py similarity index 99% rename from epochalyst/model/training/augmentation/time_series_augmentations.py rename to epochalyst/training/augmentation/time_series_augmentations.py index 8cf9759..904f5a2 100644 --- a/epochalyst/model/training/augmentation/time_series_augmentations.py +++ b/epochalyst/training/augmentation/time_series_augmentations.py @@ -187,7 +187,7 @@ def __call__(self, x: torch.Tensor) -> torch.Tensor: @dataclass -class SubstractChannels(torch.nn.Module): +class SubtractChannels(torch.nn.Module): """Randomly substract other channels from the current one.""" p: float = 0.5 diff --git a/epochalyst/model/training/augmentation/utils.py b/epochalyst/training/augmentation/utils.py similarity index 98% rename from epochalyst/model/training/augmentation/utils.py rename to epochalyst/training/augmentation/utils.py index 581702e..2dcd370 100644 --- a/epochalyst/model/training/augmentation/utils.py +++ b/epochalyst/training/augmentation/utils.py @@ -12,7 +12,7 @@ import torch -from epochalyst.model.training.utils.recursive_repr import recursive_repr +from epochalyst.training.utils.recursive_repr import recursive_repr def get_audiomentations() -> ModuleType: diff --git a/epochalyst/model/training/models/__init__.py b/epochalyst/training/models/__init__.py similarity index 100% rename from epochalyst/model/training/models/__init__.py rename to epochalyst/training/models/__init__.py diff --git a/epochalyst/model/training/models/timm.py b/epochalyst/training/models/timm.py similarity index 100% rename from epochalyst/model/training/models/timm.py rename to epochalyst/training/models/timm.py diff --git a/epochalyst/model/training/pretrain_block.py b/epochalyst/training/pretrain_block.py similarity index 100% rename from epochalyst/model/training/pretrain_block.py rename to epochalyst/training/pretrain_block.py diff --git a/epochalyst/model/training/torch_trainer.py b/epochalyst/training/torch_trainer.py similarity index 98% rename from epochalyst/model/training/torch_trainer.py rename to epochalyst/training/torch_trainer.py index 759d2d1..7f49c05 100644 --- a/epochalyst/model/training/torch_trainer.py +++ b/epochalyst/training/torch_trainer.py @@ -27,13 +27,13 @@ T_co = TypeVar("T_co", covariant=True) -def custom_collate(batch: tuple[Tensor, ...]) -> tuple[Tensor, ...]: +def custom_collate(batch: list[Tensor]) -> tuple[Tensor, Tensor]: """Collate function for the dataloader. :param batch: The batch to collate. :return: Collated batch. """ - X, y = batch + X, y = batch[0], batch[1] return X, y @@ -166,7 +166,7 @@ def log_to_terminal(self, message: str) -> None: epochs: Annotated[int, Gt(0)] = 10 patience: Annotated[int, Gt(0)] = -1 # Early stopping batch_size: Annotated[int, Gt(0)] = 32 - collate_fn: Callable[[tuple[Tensor, ...]], tuple[Tensor, ...]] = field(default=custom_collate, init=True, repr=False, compare=False) + collate_fn: Callable[[list[Tensor]], tuple[Tensor, Tensor]] = field(default=custom_collate, init=True, repr=False, compare=False) # Checkpointing checkpointing_enabled: bool = field(default=True, init=True, repr=False, compare=False) @@ -378,9 +378,7 @@ def _predict_after_train( concat_dataset, batch_size=self.batch_size, shuffle=False, - collate_fn=( - self.collate_fn if hasattr(concat_dataset, "__getitems__") else None # type: ignore[arg-type] - ), + collate_fn=(self.collate_fn if hasattr(concat_dataset, "__getitems__") else None), ) return self.predict_on_loader(pred_dataloader), y case "validation": @@ -412,7 +410,7 @@ def custom_predict(self, x: Any, **pred_args: Any) -> npt.NDArray[np.float32]: pred_dataset, batch_size=curr_batch_size, shuffle=False, - collate_fn=(self.collate_fn if hasattr(pred_dataset, "__getitems__") else None), # type: ignore[arg-type] + collate_fn=(self.collate_fn if hasattr(pred_dataset, "__getitems__") else None), ) # Predict with a single model @@ -458,9 +456,7 @@ def predict_on_loader( loader.dataset, batch_size=loader.batch_size, shuffle=False, - collate_fn=( - self.collate_fn if hasattr(loader.dataset, "__getitems__") else None # type: ignore[arg-type] - ), + collate_fn=(self.collate_fn if hasattr(loader.dataset, "__getitems__") else None), **self.dataloader_args, ) if compile_method is None: @@ -571,7 +567,7 @@ def create_dataloaders( train_dataset, batch_size=self.batch_size, shuffle=True, - collate_fn=(self.collate_fn if hasattr(train_dataset, "__getitems__") else None), # type: ignore[arg-type] + collate_fn=(self.collate_fn if hasattr(train_dataset, "__getitems__") else None), **self.dataloader_args, ) validation_loader = DataLoader( diff --git a/epochalyst/model/training/training.py b/epochalyst/training/training.py similarity index 100% rename from epochalyst/model/training/training.py rename to epochalyst/training/training.py diff --git a/epochalyst/model/training/training_block.py b/epochalyst/training/training_block.py similarity index 100% rename from epochalyst/model/training/training_block.py rename to epochalyst/training/training_block.py diff --git a/epochalyst/model/training/utils/__init__.py b/epochalyst/training/utils/__init__.py similarity index 100% rename from epochalyst/model/training/utils/__init__.py rename to epochalyst/training/utils/__init__.py diff --git a/epochalyst/model/training/utils/get_dependencies.py b/epochalyst/training/utils/get_dependencies.py similarity index 100% rename from epochalyst/model/training/utils/get_dependencies.py rename to epochalyst/training/utils/get_dependencies.py diff --git a/epochalyst/model/training/utils/recursive_repr.py b/epochalyst/training/utils/recursive_repr.py similarity index 100% rename from epochalyst/model/training/utils/recursive_repr.py rename to epochalyst/training/utils/recursive_repr.py diff --git a/epochalyst/model/training/utils/tensor_functions.py b/epochalyst/training/utils/tensor_functions.py similarity index 100% rename from epochalyst/model/training/utils/tensor_functions.py rename to epochalyst/training/utils/tensor_functions.py diff --git a/epochalyst/model/transformation/__init__.py b/epochalyst/transformation/__init__.py similarity index 100% rename from epochalyst/model/transformation/__init__.py rename to epochalyst/transformation/__init__.py diff --git a/epochalyst/model/transformation/transformation.py b/epochalyst/transformation/transformation.py similarity index 100% rename from epochalyst/model/transformation/transformation.py rename to epochalyst/transformation/transformation.py diff --git a/epochalyst/model/transformation/transformation_block.py b/epochalyst/transformation/transformation_block.py similarity index 100% rename from epochalyst/model/transformation/transformation_block.py rename to epochalyst/transformation/transformation_block.py diff --git a/tests/model/transformation/__init__.py b/tests/model/transformation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/model/test_ensemble.py b/tests/test_ensemble.py similarity index 94% rename from tests/model/test_ensemble.py rename to tests/test_ensemble.py index 73243b9..b998bc9 100644 --- a/tests/model/test_ensemble.py +++ b/tests/test_ensemble.py @@ -5,10 +5,10 @@ import numpy as np import pytest -from epochalyst.model import EnsemblePipeline -from epochalyst.model import ModelPipeline -from epochalyst.model import TransformationPipeline -from epochalyst.model import TransformationBlock +from epochalyst import EnsemblePipeline +from epochalyst import ModelPipeline +from epochalyst.transformation import TransformationPipeline +from epochalyst.transformation import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/model/test_model.py b/tests/test_model.py similarity index 92% rename from tests/model/test_model.py rename to tests/test_model.py index 8ca7201..f33554b 100644 --- a/tests/model/test_model.py +++ b/tests/test_model.py @@ -5,9 +5,9 @@ import numpy as np import pytest -from epochalyst.model.model import ModelPipeline -from epochalyst.model.transformation.transformation import TransformationPipeline -from epochalyst.model.transformation.transformation_block import TransformationBlock +from epochalyst import ModelPipeline +from epochalyst.transformation import TransformationPipeline +from epochalyst.transformation import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/model/__init__.py b/tests/training/__init__.py similarity index 100% rename from tests/model/__init__.py rename to tests/training/__init__.py diff --git a/tests/model/training/__init__.py b/tests/training/augmentation/__init__.py similarity index 100% rename from tests/model/training/__init__.py rename to tests/training/augmentation/__init__.py diff --git a/tests/model/training/augmentation/test_audio/white_noise.wav b/tests/training/augmentation/test_audio/white_noise.wav similarity index 100% rename from tests/model/training/augmentation/test_audio/white_noise.wav rename to tests/training/augmentation/test_audio/white_noise.wav diff --git a/tests/model/training/augmentation/test_image_augmentations.py b/tests/training/augmentation/test_image_augmentations.py similarity index 96% rename from tests/model/training/augmentation/test_image_augmentations.py rename to tests/training/augmentation/test_image_augmentations.py index 149dd22..f7a53e3 100644 --- a/tests/model/training/augmentation/test_image_augmentations.py +++ b/tests/training/augmentation/test_image_augmentations.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model import image_augmentations +from epochalyst.training.augmentation import image_augmentations class TestImageAugmentations: diff --git a/tests/model/training/augmentation/test_time_series_augmentations.py b/tests/training/augmentation/test_time_series_augmentations.py similarity index 96% rename from tests/model/training/augmentation/test_time_series_augmentations.py rename to tests/training/augmentation/test_time_series_augmentations.py index 144cf9c..662ea7e 100644 --- a/tests/model/training/augmentation/test_time_series_augmentations.py +++ b/tests/training/augmentation/test_time_series_augmentations.py @@ -1,7 +1,7 @@ import numpy as np import torch -from epochalyst.model import time_series_augmentations +from epochalyst.training.augmentation import time_series_augmentations def set_torch_seed(seed: int = 42) -> None: @@ -152,7 +152,7 @@ def test_reverse_1d(self): def test_subtract_channels(self): set_torch_seed(42) - subtract_channels = time_series_augmentations.SubstractChannels(p=1.0) + subtract_channels = time_series_augmentations.SubtractChannels(p=1.0) # Only works for multi-channel sequences x = torch.ones(32, 2, 100) augmented_x = subtract_channels(x) @@ -162,7 +162,7 @@ def test_subtract_channels(self): assert torch.allclose(torch.zeros(*augmented_x.shape), augmented_x) - subtract_channels = time_series_augmentations.SubstractChannels(p=0) + subtract_channels = time_series_augmentations.SubtractChannels(p=0) augmented_x = subtract_channels(x) assert torch.all(augmented_x == x) @@ -205,7 +205,7 @@ def test_energy_cutmix(self): def test_add_background_noise_wrapper(self): add_background_noise_wrapper = time_series_augmentations.AddBackgroundNoiseWrapper(p=1.0, - sounds_path='tests/pipeline/model/training/augmentation/test_audio/white_noise.wav') + sounds_path='tests/training/augmentation/test_audio/white_noise.wav') x = torch.rand(44100, dtype=torch.float32) sr = 44100 augmented_x = add_background_noise_wrapper(x, sr) diff --git a/tests/model/training/augmentation/test_utils.py b/tests/training/augmentation/test_utils.py similarity index 98% rename from tests/model/training/augmentation/test_utils.py rename to tests/training/augmentation/test_utils.py index be40858..d62ff2f 100644 --- a/tests/model/training/augmentation/test_utils.py +++ b/tests/training/augmentation/test_utils.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model import utils +from epochalyst.training.augmentation import utils def set_torch_seed(seed: int = 42) -> None: diff --git a/tests/model/training/augmentation/__init__.py b/tests/training/models/__init__.py similarity index 100% rename from tests/model/training/augmentation/__init__.py rename to tests/training/models/__init__.py diff --git a/tests/model/training/models/test_timm.py b/tests/training/models/test_timm.py similarity index 86% rename from tests/model/training/models/test_timm.py rename to tests/training/models/test_timm.py index ffde7c0..ab847d7 100644 --- a/tests/model/training/models/test_timm.py +++ b/tests/training/models/test_timm.py @@ -1,6 +1,6 @@ import torch -from epochalyst.model import Timm +from epochalyst.training.models import Timm class TestTimm: diff --git a/tests/model/training/test_pretrain_block.py b/tests/training/test_pretrain_block.py similarity index 90% rename from tests/model/training/test_pretrain_block.py rename to tests/training/test_pretrain_block.py index 5d554eb..afe0e43 100644 --- a/tests/model/training/test_pretrain_block.py +++ b/tests/training/test_pretrain_block.py @@ -1,6 +1,6 @@ import pytest -from epochalyst.model.training.pretrain_block import PretrainBlock +from epochalyst.training import PretrainBlock class TestPretrainBlock: diff --git a/tests/model/training/test_torch_trainer.py b/tests/training/test_torch_trainer.py similarity index 99% rename from tests/model/training/test_torch_trainer.py rename to tests/training/test_torch_trainer.py index 589229c..4e88d58 100644 --- a/tests/model/training/test_torch_trainer.py +++ b/tests/training/test_torch_trainer.py @@ -4,9 +4,9 @@ import time from dataclasses import dataclass -from epochalyst.model.training._custom_data_parallel import _CustomDataParallel +from epochalyst.training._custom_data_parallel import _CustomDataParallel -from epochalyst.model.training.torch_trainer import custom_collate +from epochalyst.training.torch_trainer import custom_collate from typing import Any from unittest.mock import patch @@ -14,7 +14,7 @@ import pytest import torch -from epochalyst.model.training.torch_trainer import TorchTrainer +from epochalyst.training.torch_trainer import TorchTrainer from tests.constants import TEMP_DIR diff --git a/tests/model/training/test_training.py b/tests/training/test_training.py similarity index 97% rename from tests/model/training/test_training.py rename to tests/training/test_training.py index 8d1389b..ec495b3 100644 --- a/tests/model/training/test_training.py +++ b/tests/training/test_training.py @@ -2,8 +2,8 @@ import pytest from agogos.training import Trainer -from epochalyst.model.training.training import TrainingPipeline -from epochalyst.model.training.training_block import TrainingBlock +from epochalyst.training import TrainingPipeline +from epochalyst.training import TrainingBlock from tests.constants import TEMP_DIR diff --git a/tests/model/training/test_training_block.py b/tests/training/test_training_block.py similarity index 96% rename from tests/model/training/test_training_block.py rename to tests/training/test_training_block.py index 4e4343c..accadb0 100644 --- a/tests/model/training/test_training_block.py +++ b/tests/training/test_training_block.py @@ -2,7 +2,7 @@ import pytest -from epochalyst.model.training.training_block import TrainingBlock +from epochalyst.training import TrainingBlock TEMP_DIR = Path("tests/temp") diff --git a/tests/model/training/models/__init__.py b/tests/training/utils/__init__.py similarity index 100% rename from tests/model/training/models/__init__.py rename to tests/training/utils/__init__.py diff --git a/tests/model/training/utils/test_get_dependencies.py b/tests/training/utils/test_get_dependencies.py similarity index 96% rename from tests/model/training/utils/test_get_dependencies.py rename to tests/training/utils/test_get_dependencies.py index 70823a6..2f91c22 100644 --- a/tests/model/training/utils/test_get_dependencies.py +++ b/tests/training/utils/test_get_dependencies.py @@ -3,7 +3,7 @@ import pytest -from epochalyst.model.training.utils import _get_openvino, _get_onnxrt +from epochalyst.training.utils import _get_openvino, _get_onnxrt class TestGetDependencies: diff --git a/tests/model/test_recursive_repr.py b/tests/training/utils/test_recursive_repr.py similarity index 94% rename from tests/model/test_recursive_repr.py rename to tests/training/utils/test_recursive_repr.py index 7e6f6b7..81bcb75 100644 --- a/tests/model/test_recursive_repr.py +++ b/tests/training/utils/test_recursive_repr.py @@ -1,4 +1,4 @@ -from epochalyst.model.training.utils.recursive_repr import recursive_repr +from epochalyst.training.utils.recursive_repr import recursive_repr class TestRecursiveRepr: diff --git a/tests/model/training/utils/__init__.py b/tests/transformation/__init__.py similarity index 100% rename from tests/model/training/utils/__init__.py rename to tests/transformation/__init__.py diff --git a/tests/model/transformation/test_transformation.py b/tests/transformation/test_transformation.py similarity index 96% rename from tests/model/transformation/test_transformation.py rename to tests/transformation/test_transformation.py index afdbf9b..453462f 100644 --- a/tests/model/transformation/test_transformation.py +++ b/tests/transformation/test_transformation.py @@ -5,8 +5,8 @@ import pytest from agogos.transforming import Transformer -from epochalyst.model.transformation.transformation import TransformationPipeline -from epochalyst.model.transformation.transformation_block import TransformationBlock +from epochalyst.transformation import TransformationPipeline +from epochalyst.transformation import TransformationBlock from tests.constants import TEMP_DIR diff --git a/tests/model/transformation/test_transformation_block.py b/tests/transformation/test_transformation_block.py similarity index 97% rename from tests/model/transformation/test_transformation_block.py rename to tests/transformation/test_transformation_block.py index ee1df73..710b580 100644 --- a/tests/model/transformation/test_transformation_block.py +++ b/tests/transformation/test_transformation_block.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from epochalyst.model.transformation.transformation_block import TransformationBlock +from epochalyst.transformation import TransformationBlock TEMP_DIR = Path("tests/temp") From 65248b8be98c1831d378892b98326449958b754b Mon Sep 17 00:00:00 2001 From: Emiel Witting Date: Fri, 28 Jun 2024 11:02:54 +0200 Subject: [PATCH 5/7] remove utils dir --- epochalyst/caching/__init__.py | 5 +++++ epochalyst/{utils => caching}/cacher.py | 2 +- epochalyst/ensemble.py | 2 +- epochalyst/logging/__init__.py | 5 +++++ epochalyst/{utils => logging}/logger.py | 0 epochalyst/model.py | 2 +- epochalyst/training/training.py | 2 +- epochalyst/training/training_block.py | 2 +- epochalyst/transformation/transformation.py | 2 +- epochalyst/transformation/transformation_block.py | 2 +- epochalyst/utils/__init__.py | 6 ------ tests/{utils => caching}/__init__.py | 0 tests/{utils => caching}/test__cacher.py | 4 ++-- tests/logging/__init__.py | 0 tests/{utils => logging}/test_logger.py | 2 +- 15 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 epochalyst/caching/__init__.py rename epochalyst/{utils => caching}/cacher.py (99%) create mode 100644 epochalyst/logging/__init__.py rename epochalyst/{utils => logging}/logger.py (100%) delete mode 100644 epochalyst/utils/__init__.py rename tests/{utils => caching}/__init__.py (100%) rename tests/{utils => caching}/test__cacher.py (99%) create mode 100644 tests/logging/__init__.py rename tests/{utils => logging}/test_logger.py (97%) diff --git a/epochalyst/caching/__init__.py b/epochalyst/caching/__init__.py new file mode 100644 index 0000000..c4f989f --- /dev/null +++ b/epochalyst/caching/__init__.py @@ -0,0 +1,5 @@ +"""Caching module for epochalyst.""" + +from .cacher import CacheArgs, Cacher + +__all__ = ["Cacher", "CacheArgs"] diff --git a/epochalyst/utils/cacher.py b/epochalyst/caching/cacher.py similarity index 99% rename from epochalyst/utils/cacher.py rename to epochalyst/caching/cacher.py index 2f4cd1b..32c4eb1 100644 --- a/epochalyst/utils/cacher.py +++ b/epochalyst/caching/cacher.py @@ -8,7 +8,7 @@ from typing_extensions import NotRequired -from .logger import Logger +from epochalyst.logging import Logger try: import dask.array as da diff --git a/epochalyst/ensemble.py b/epochalyst/ensemble.py index 370ee78..400986c 100644 --- a/epochalyst/ensemble.py +++ b/epochalyst/ensemble.py @@ -4,7 +4,7 @@ from agogos.training import ParallelTrainingSystem -from epochalyst.utils import CacheArgs +from epochalyst.caching import CacheArgs class EnsemblePipeline(ParallelTrainingSystem): diff --git a/epochalyst/logging/__init__.py b/epochalyst/logging/__init__.py new file mode 100644 index 0000000..bfe1d70 --- /dev/null +++ b/epochalyst/logging/__init__.py @@ -0,0 +1,5 @@ +"""Logging module, contains Logger class for logging messages to console and file.""" + +from .logger import Logger + +__all__ = ["Logger"] diff --git a/epochalyst/utils/logger.py b/epochalyst/logging/logger.py similarity index 100% rename from epochalyst/utils/logger.py rename to epochalyst/logging/logger.py diff --git a/epochalyst/model.py b/epochalyst/model.py index 3687e36..f91b6c7 100644 --- a/epochalyst/model.py +++ b/epochalyst/model.py @@ -4,7 +4,7 @@ from agogos.training import Pipeline -from epochalyst.utils import CacheArgs +from epochalyst.caching import CacheArgs class ModelPipeline(Pipeline): diff --git a/epochalyst/training/training.py b/epochalyst/training/training.py index e8c8471..5453719 100644 --- a/epochalyst/training/training.py +++ b/epochalyst/training/training.py @@ -4,7 +4,7 @@ from agogos.training import TrainingSystem, TrainType -from epochalyst.utils import CacheArgs, Cacher +from epochalyst.caching import CacheArgs, Cacher class TrainingPipeline(TrainingSystem, Cacher): diff --git a/epochalyst/training/training_block.py b/epochalyst/training/training_block.py index d13089e..d64cf8f 100644 --- a/epochalyst/training/training_block.py +++ b/epochalyst/training/training_block.py @@ -5,7 +5,7 @@ from agogos.training import Trainer -from epochalyst.utils import CacheArgs, Cacher +from epochalyst.caching import CacheArgs, Cacher class TrainingBlock(Trainer, Cacher): diff --git a/epochalyst/transformation/transformation.py b/epochalyst/transformation/transformation.py index 6bb7bda..cbd342c 100644 --- a/epochalyst/transformation/transformation.py +++ b/epochalyst/transformation/transformation.py @@ -5,7 +5,7 @@ from agogos.transforming import TransformingSystem, TransformType -from epochalyst.utils.cacher import CacheArgs, Cacher +from epochalyst.caching.cacher import CacheArgs, Cacher @dataclass diff --git a/epochalyst/transformation/transformation_block.py b/epochalyst/transformation/transformation_block.py index 1d5c215..a163c7b 100644 --- a/epochalyst/transformation/transformation_block.py +++ b/epochalyst/transformation/transformation_block.py @@ -5,7 +5,7 @@ from agogos.transforming import Transformer -from epochalyst.utils.cacher import CacheArgs, Cacher +from epochalyst.caching.cacher import CacheArgs, Cacher class TransformationBlock(Transformer, Cacher): diff --git a/epochalyst/utils/__init__.py b/epochalyst/utils/__init__.py deleted file mode 100644 index 8069675..0000000 --- a/epochalyst/utils/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Utils module, contains the Cacher and Logger classes.""" - -from .cacher import CacheArgs, Cacher -from .logger import Logger - -__all__ = ["CacheArgs", "Cacher", "Logger"] diff --git a/tests/utils/__init__.py b/tests/caching/__init__.py similarity index 100% rename from tests/utils/__init__.py rename to tests/caching/__init__.py diff --git a/tests/utils/test__cacher.py b/tests/caching/test__cacher.py similarity index 99% rename from tests/utils/test__cacher.py rename to tests/caching/test__cacher.py index 125158e..f119c28 100644 --- a/tests/utils/test__cacher.py +++ b/tests/caching/test__cacher.py @@ -5,8 +5,8 @@ import polars as pl import pytest -from epochalyst.utils.cacher import Cacher -from epochalyst.utils.logger import Logger +from epochalyst.caching.cacher import Cacher +from epochalyst.logging.logger import Logger from tests.constants import TEMP_DIR diff --git a/tests/logging/__init__.py b/tests/logging/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/test_logger.py b/tests/logging/test_logger.py similarity index 97% rename from tests/utils/test_logger.py rename to tests/logging/test_logger.py index 0034ee8..cfb8ca5 100644 --- a/tests/utils/test_logger.py +++ b/tests/logging/test_logger.py @@ -3,7 +3,7 @@ import pytest -from epochalyst.utils.logger import Logger +from epochalyst.logging.logger import Logger test_string = "Test" From dd854be9b9ef0c7a04a77d2344384bd4ae54a057 Mon Sep 17 00:00:00 2001 From: Jeffrey Lim Date: Fri, 28 Jun 2024 17:25:14 +0200 Subject: [PATCH 6/7] Update cacher.py --- epochalyst/caching/cacher.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/epochalyst/caching/cacher.py b/epochalyst/caching/cacher.py index 32c4eb1..6a4dcc3 100644 --- a/epochalyst/caching/cacher.py +++ b/epochalyst/caching/cacher.py @@ -6,8 +6,6 @@ import sys from typing import Any, Literal, TypedDict -from typing_extensions import NotRequired - from epochalyst.logging import Logger try: @@ -32,9 +30,9 @@ """User doesn't require these packages""" if sys.version_info < (3, 11): # pragma: no cover ( Date: Fri, 28 Jun 2024 17:27:11 +0200 Subject: [PATCH 7/7] Update __init__.py --- epochalyst/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/epochalyst/__init__.py b/epochalyst/__init__.py index 8662c7c..64eaef1 100644 --- a/epochalyst/__init__.py +++ b/epochalyst/__init__.py @@ -1,8 +1,4 @@ -"""The epochalyst package. - -It consists of the following modules: - -""" +"""The epochalyst package.""" from .ensemble import EnsemblePipeline from .model import ModelPipeline