From 3557fdc4297abdd80a98052dedd8943b423effa8 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 19:41:04 +0100 Subject: [PATCH 1/8] ruff: prune & sort imports --- ampel/abstract/AbsAlertLoader.py | 11 ++++--- ampel/abstract/AbsAlertRegister.py | 2 +- ampel/abstract/AbsAlertSupplier.py | 7 +++-- ampel/alert/AlertConsumer.py | 31 ++++++++++--------- ampel/alert/AlertConsumerError.py | 1 + ampel/alert/AlertFileList.py | 4 ++- ampel/alert/BaseAlertSupplier.py | 11 ++++--- ampel/alert/FilterBlock.py | 24 ++++++++------ ampel/alert/FilterBlocksHandler.py | 3 +- ampel/alert/FilteringAlertSupplier.py | 5 +-- ampel/alert/filter/BasicMultiFilter.py | 8 ++--- ampel/alert/load/DirAlertLoader.py | 4 ++- ampel/alert/load/DirFileNamesLoader.py | 4 ++- ampel/alert/load/DirTaggedAlertLoader.py | 3 +- ampel/alert/load/FileAlertLoader.py | 2 +- ampel/alert/load/TarAlertLoader.py | 5 +-- ampel/alert/load/TarballWalker.py | 1 + ampel/alert/reject/BaseAlertRegister.py | 9 +++--- ampel/alert/reject/DBRejectedLogsHandler.py | 9 +++--- ampel/alert/reject/FullActiveAlertRegister.py | 7 +++-- ampel/alert/reject/FullAlertRegister.py | 9 +++--- .../reject/GeneralActiveAlertRegister.py | 5 +-- ampel/alert/reject/GeneralAlertRegister.py | 5 +-- .../reject/MinimalActiveAlertRegister.py | 5 +-- ampel/alert/reject/MinimalAlertRegister.py | 7 +++-- ampel/dev/AutoCompleteBenchmark.py | 10 +++--- ampel/dev/UnitTestAlertSupplier.py | 2 +- ampel/model/AlertConsumerModel.py | 9 +++--- ampel/template/AbsEasyChannelTemplate.py | 14 +++++---- setup.py | 2 +- tests/conftest.py | 8 ++--- tests/test_AbsAlertLoader.py | 5 +-- tests/test_AbsEasyChannelTemplate.py | 7 +++-- tests/test_AlertConsumer.py | 17 +++++----- tests/test_EasyAlertConsumerTemplate.py | 15 +++++---- 35 files changed, 155 insertions(+), 116 deletions(-) diff --git a/ampel/abstract/AbsAlertLoader.py b/ampel/abstract/AbsAlertLoader.py index f27cd67..fa811cb 100755 --- a/ampel/abstract/AbsAlertLoader.py +++ b/ampel/abstract/AbsAlertLoader.py @@ -7,14 +7,15 @@ # Last Modified Date: 19.12.2022 # Last Modified By: valery brinnel -from ampel.types import T -from typing import Generic from collections.abc import Iterator -from ampel.struct.Resource import Resource -from ampel.log.AmpelLogger import AmpelLogger +from typing import Generic + from ampel.base.AmpelABC import AmpelABC -from ampel.base.decorator import abstractmethod from ampel.base.AmpelBaseModel import AmpelBaseModel +from ampel.base.decorator import abstractmethod +from ampel.log.AmpelLogger import AmpelLogger +from ampel.struct.Resource import Resource +from ampel.types import T class AbsAlertLoader(AmpelABC, AmpelBaseModel, Generic[T], abstract=True): diff --git a/ampel/abstract/AbsAlertRegister.py b/ampel/abstract/AbsAlertRegister.py index b578698..246f967 100644 --- a/ampel/abstract/AbsAlertRegister.py +++ b/ampel/abstract/AbsAlertRegister.py @@ -9,8 +9,8 @@ from ampel.base.AmpelABC import AmpelABC from ampel.base.decorator import abstractmethod -from ampel.core.ContextUnit import ContextUnit from ampel.core.AmpelRegister import AmpelRegister +from ampel.core.ContextUnit import ContextUnit from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol diff --git a/ampel/abstract/AbsAlertSupplier.py b/ampel/abstract/AbsAlertSupplier.py index 109caf4..1b2e885 100755 --- a/ampel/abstract/AbsAlertSupplier.py +++ b/ampel/abstract/AbsAlertSupplier.py @@ -8,12 +8,13 @@ # Last Modified By: valery brinnel from typing import Iterator -from ampel.log.AmpelLogger import AmpelLogger + from ampel.base.AmpelABC import AmpelABC -from ampel.base.decorator import abstractmethod from ampel.base.AmpelUnit import AmpelUnit -from ampel.struct.Resource import Resource +from ampel.base.decorator import abstractmethod +from ampel.log.AmpelLogger import AmpelLogger from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from ampel.struct.Resource import Resource class AbsAlertSupplier(AmpelUnit, AmpelABC, abstract=True): diff --git a/ampel/alert/AlertConsumer.py b/ampel/alert/AlertConsumer.py index 46c49c7..96e5118 100755 --- a/ampel/alert/AlertConsumer.py +++ b/ampel/alert/AlertConsumer.py @@ -8,32 +8,33 @@ # Last Modified By: valery brinnel import sys -from typing import Any -from typing_extensions import Self from collections.abc import Sequence +from signal import SIGINT, SIGTERM, default_int_handler, signal +from typing import Any + from pymongo.errors import PyMongoError -from signal import signal, SIGINT, SIGTERM, default_int_handler +from typing_extensions import Self -from ampel.core.AmpelContext import AmpelContext -from ampel.enum.EventCode import EventCode -from ampel.model.UnitModel import UnitModel -from ampel.core.EventHandler import EventHandler from ampel.abstract.AbsAlertSupplier import AbsAlertSupplier from ampel.abstract.AbsEventUnit import AbsEventUnit -from ampel.base.AuxUnitRegister import AuxUnitRegister +from ampel.alert.AlertConsumerError import AlertConsumerError +from ampel.alert.AlertConsumerMetrics import AlertConsumerMetrics, stat_time from ampel.alert.FilterBlocksHandler import FilterBlocksHandler +from ampel.base.AuxUnitRegister import AuxUnitRegister +from ampel.core.AmpelContext import AmpelContext +from ampel.core.EventHandler import EventHandler +from ampel.enum.EventCode import EventCode from ampel.ingest.ChainedIngestionHandler import ChainedIngestionHandler -from ampel.mongo.update.DBUpdatesBuffer import DBUpdatesBuffer -from ampel.log import AmpelLogger, LogFlag, VERBOSE -from ampel.log.utils import report_exception +from ampel.log import VERBOSE, AmpelLogger, LogFlag from ampel.log.AmpelLoggingError import AmpelLoggingError from ampel.log.LightLogRecord import LightLogRecord -from ampel.alert.AlertConsumerError import AlertConsumerError -from ampel.alert.AlertConsumerMetrics import AlertConsumerMetrics, stat_time -from ampel.model.ingest.CompilerOptions import CompilerOptions +from ampel.log.utils import report_exception from ampel.model.AlertConsumerModel import AlertConsumerModel -from ampel.util.mappings import get_by_path, merge_dict +from ampel.model.ingest.CompilerOptions import CompilerOptions +from ampel.model.UnitModel import UnitModel +from ampel.mongo.update.DBUpdatesBuffer import DBUpdatesBuffer from ampel.util.freeze import recursive_unfreeze +from ampel.util.mappings import get_by_path, merge_dict class AlertConsumer(AbsEventUnit, AlertConsumerModel): diff --git a/ampel/alert/AlertConsumerError.py b/ampel/alert/AlertConsumerError.py index 085da25..30af29c 100644 --- a/ampel/alert/AlertConsumerError.py +++ b/ampel/alert/AlertConsumerError.py @@ -10,6 +10,7 @@ import signal from enum import IntEnum + class AlertConsumerError(IntEnum): CONNECTIVITY = 1 SIGINT = signal.SIGINT # 2 diff --git a/ampel/alert/AlertFileList.py b/ampel/alert/AlertFileList.py index 5ac3e6f..7e291a9 100644 --- a/ampel/alert/AlertFileList.py +++ b/ampel/alert/AlertFileList.py @@ -9,6 +9,7 @@ import logging + # pylint: disable=logging-format-interpolation class AlertFileList: @@ -59,7 +60,8 @@ def build_file_list(self): """ """ self.logger.debug("Building internal file list") - import glob, os + import glob + import os all_files = sorted(glob.glob(self.folder + "/" + self.extension), key=os.path.getmtime) if self.min_index is not None: diff --git a/ampel/alert/BaseAlertSupplier.py b/ampel/alert/BaseAlertSupplier.py index 1b8cef8..4bba6a8 100755 --- a/ampel/alert/BaseAlertSupplier.py +++ b/ampel/alert/BaseAlertSupplier.py @@ -8,16 +8,17 @@ # Last Modified By: valery brinnel import json +from collections.abc import Callable, Iterator from io import IOBase from typing import Any, Literal -from collections.abc import Callable, Iterator -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol + +from ampel.abstract.AbsAlertLoader import AbsAlertLoader from ampel.abstract.AbsAlertSupplier import AbsAlertSupplier -from ampel.log.AmpelLogger import AmpelLogger -from ampel.base.decorator import abstractmethod from ampel.base.AuxUnitRegister import AuxUnitRegister -from ampel.abstract.AbsAlertLoader import AbsAlertLoader +from ampel.base.decorator import abstractmethod +from ampel.log.AmpelLogger import AmpelLogger from ampel.model.UnitModel import UnitModel +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol from ampel.struct.Resource import Resource diff --git a/ampel/alert/FilterBlock.py b/ampel/alert/FilterBlock.py index 283f2e2..812bcb7 100755 --- a/ampel/alert/FilterBlock.py +++ b/ampel/alert/FilterBlock.py @@ -7,22 +7,28 @@ # Last Modified Date: 27.06.2022 # Last Modified By: valery brinnel +from collections.abc import Callable from logging import LogRecord from typing import Any, cast -from collections.abc import Callable -from ampel.types import ChannelId, StockId + +from ampel.abstract.AbsAlertFilter import AbsAlertFilter +from ampel.abstract.AbsAlertRegister import AbsAlertRegister +from ampel.alert.AlertConsumerMetrics import ( + stat_accepted, + stat_autocomplete, + stat_rejected, + stat_time, +) from ampel.core.AmpelContext import AmpelContext -from ampel.model.ingest.FilterModel import FilterModel -from ampel.log.AmpelLogger import AmpelLogger, INFO -from ampel.log.handlers.EnclosedChanRecordBufHandler import EnclosedChanRecordBufHandler +from ampel.log.AmpelLogger import INFO, AmpelLogger from ampel.log.handlers.ChanRecordBufHandler import ChanRecordBufHandler +from ampel.log.handlers.EnclosedChanRecordBufHandler import EnclosedChanRecordBufHandler from ampel.log.LightLogRecord import LightLogRecord from ampel.log.LogFlag import LogFlag -from ampel.protocol.LoggingHandlerProtocol import LoggingHandlerProtocol -from ampel.abstract.AbsAlertFilter import AbsAlertFilter -from ampel.abstract.AbsAlertRegister import AbsAlertRegister -from ampel.alert.AlertConsumerMetrics import stat_accepted, stat_rejected, stat_autocomplete, stat_time +from ampel.model.ingest.FilterModel import FilterModel from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from ampel.protocol.LoggingHandlerProtocol import LoggingHandlerProtocol +from ampel.types import ChannelId, StockId def no_filter(alert: Any) -> bool: diff --git a/ampel/alert/FilterBlocksHandler.py b/ampel/alert/FilterBlocksHandler.py index 4439605..b33d4ad 100755 --- a/ampel/alert/FilterBlocksHandler.py +++ b/ampel/alert/FilterBlocksHandler.py @@ -8,11 +8,12 @@ # Last Modified By: valery brinnel from collections.abc import Sequence + from ampel.alert.FilterBlock import FilterBlock from ampel.core.AmpelContext import AmpelContext from ampel.log.AmpelLogger import AmpelLogger -from ampel.model.ingest.IngestDirective import IngestDirective from ampel.model.ingest.DualIngestDirective import DualIngestDirective +from ampel.model.ingest.IngestDirective import IngestDirective class FilterBlocksHandler: diff --git a/ampel/alert/FilteringAlertSupplier.py b/ampel/alert/FilteringAlertSupplier.py index 35c4027..352f307 100755 --- a/ampel/alert/FilteringAlertSupplier.py +++ b/ampel/alert/FilteringAlertSupplier.py @@ -8,10 +8,11 @@ # Last Modified By: valery brinnel from typing import Iterator + from ampel.abstract.AbsAlertSupplier import AbsAlertSupplier -from ampel.model.UnitModel import UnitModel -from ampel.log.AmpelLogger import AmpelLogger from ampel.base.AuxUnitRegister import AuxUnitRegister +from ampel.log.AmpelLogger import AmpelLogger +from ampel.model.UnitModel import UnitModel from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol diff --git a/ampel/alert/filter/BasicMultiFilter.py b/ampel/alert/filter/BasicMultiFilter.py index 34eb9af..9f69c06 100755 --- a/ampel/alert/filter/BasicMultiFilter.py +++ b/ampel/alert/filter/BasicMultiFilter.py @@ -8,12 +8,12 @@ # Last Modified By: Jakob van Santen import operator -from ampel.abstract.AbsAlertFilter import AbsAlertFilter -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from collections.abc import Callable, Sequence +from typing import ClassVar, Literal +from ampel.abstract.AbsAlertFilter import AbsAlertFilter from ampel.base.AmpelBaseModel import AmpelBaseModel -from typing import Literal, ClassVar -from collections.abc import Callable, Sequence +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class PhotoAlertQuery(AmpelBaseModel): diff --git a/ampel/alert/load/DirAlertLoader.py b/ampel/alert/load/DirAlertLoader.py index 57e17c7..c4b8fef 100755 --- a/ampel/alert/load/DirAlertLoader.py +++ b/ampel/alert/load/DirAlertLoader.py @@ -8,6 +8,7 @@ # Last Modified By: valery brinnel from io import BytesIO, StringIO + from ampel.abstract.AbsAlertLoader import AbsAlertLoader @@ -58,7 +59,8 @@ def build_file_list(self) -> None: self.logger.debug("Building internal file list") - import glob, os + import glob + import os all_files = sorted( glob.glob( os.path.join(self.folder, "*." + self.extension) diff --git a/ampel/alert/load/DirFileNamesLoader.py b/ampel/alert/load/DirFileNamesLoader.py index 8dedf1c..d2696ce 100755 --- a/ampel/alert/load/DirFileNamesLoader.py +++ b/ampel/alert/load/DirFileNamesLoader.py @@ -7,7 +7,9 @@ # Last Modified Date: 25.10.2021 # Last Modified By: valery brinnel -import glob, os +import glob +import os + from ampel.abstract.AbsAlertLoader import AbsAlertLoader diff --git a/ampel/alert/load/DirTaggedAlertLoader.py b/ampel/alert/load/DirTaggedAlertLoader.py index 4476766..7a39d42 100755 --- a/ampel/alert/load/DirTaggedAlertLoader.py +++ b/ampel/alert/load/DirTaggedAlertLoader.py @@ -7,8 +7,9 @@ # Last Modified Date: 04.10.2021 # Last Modified By: valery brinnel -from os.path import basename from io import BytesIO, StringIO +from os.path import basename + from ampel.alert.load.DirAlertLoader import DirAlertLoader diff --git a/ampel/alert/load/FileAlertLoader.py b/ampel/alert/load/FileAlertLoader.py index 52a939a..18d1c96 100755 --- a/ampel/alert/load/FileAlertLoader.py +++ b/ampel/alert/load/FileAlertLoader.py @@ -8,7 +8,7 @@ # Last Modified By: valery brinnel from io import BytesIO -from typing import List + from ampel.abstract.AbsAlertLoader import AbsAlertLoader diff --git a/ampel/alert/load/TarAlertLoader.py b/ampel/alert/load/TarAlertLoader.py index 8fd8b2d..e43baf6 100755 --- a/ampel/alert/load/TarAlertLoader.py +++ b/ampel/alert/load/TarAlertLoader.py @@ -9,10 +9,11 @@ import tarfile from gzip import GzipFile -from typing import IO, TypeAlias, TYPE_CHECKING +from typing import IO, TYPE_CHECKING, TypeAlias + +from ampel.abstract.AbsAlertLoader import AbsAlertLoader from ampel.log.AmpelLogger import AmpelLogger from ampel.types import Traceless -from ampel.abstract.AbsAlertLoader import AbsAlertLoader # use IOBase at runtime, because isinstance(anything, IO[bytes]) is always # False. diff --git a/ampel/alert/load/TarballWalker.py b/ampel/alert/load/TarballWalker.py index bf25e3d..e538ad8 100644 --- a/ampel/alert/load/TarballWalker.py +++ b/ampel/alert/load/TarballWalker.py @@ -9,6 +9,7 @@ import tarfile + class TarballWalker: """ """ diff --git a/ampel/alert/reject/BaseAlertRegister.py b/ampel/alert/reject/BaseAlertRegister.py index cc26d88..a6b8c73 100755 --- a/ampel/alert/reject/BaseAlertRegister.py +++ b/ampel/alert/reject/BaseAlertRegister.py @@ -7,12 +7,13 @@ # Last Modified Date: 31.08.2020 # Last Modified By: valery brinnel -from typing import BinaryIO, Literal, Any, ClassVar -from collections.abc import Sequence, Generator -from ampel.types import ChannelId +from collections.abc import Generator, Sequence +from typing import Any, BinaryIO, ClassVar, Literal + from ampel.abstract.AbsAlertRegister import AbsAlertRegister -from ampel.util.register import find, reg_iter from ampel.log import VERBOSE +from ampel.types import ChannelId +from ampel.util.register import find, reg_iter class BaseAlertRegister(AbsAlertRegister, abstract=True): diff --git a/ampel/alert/reject/DBRejectedLogsHandler.py b/ampel/alert/reject/DBRejectedLogsHandler.py index 8225b8f..5d4ba34 100755 --- a/ampel/alert/reject/DBRejectedLogsHandler.py +++ b/ampel/alert/reject/DBRejectedLogsHandler.py @@ -7,18 +7,19 @@ # Last Modified Date: 09.05.2020 # Last Modified By: valery brinnel -from time import time from logging import DEBUG, WARNING, LogRecord +from time import time from typing import Any + from pymongo.errors import BulkWriteError from pymongo.operations import UpdateOne -from ampel.types import ChannelId +from ampel.core.ContextUnit import ContextUnit from ampel.log.AmpelLogger import AmpelLogger -from ampel.log.LightLogRecord import LightLogRecord from ampel.log.AmpelLoggingError import AmpelLoggingError +from ampel.log.LightLogRecord import LightLogRecord from ampel.log.LoggingErrorReporter import LoggingErrorReporter -from ampel.core.ContextUnit import ContextUnit +from ampel.types import ChannelId class DBRejectedLogsHandler(ContextUnit): diff --git a/ampel/alert/reject/FullActiveAlertRegister.py b/ampel/alert/reject/FullActiveAlertRegister.py index 1a0934e..678716a 100755 --- a/ampel/alert/reject/FullActiveAlertRegister.py +++ b/ampel/alert/reject/FullActiveAlertRegister.py @@ -7,12 +7,13 @@ # Last Modified Date: 27.06.2022 # Last Modified By: valery brinnel -from time import time +from collections.abc import Sequence from struct import pack +from time import time from typing import ClassVar -from collections.abc import Sequence -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol + from ampel.alert.reject.FullAlertRegister import FullAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class FullActiveAlertRegister(FullAlertRegister): diff --git a/ampel/alert/reject/FullAlertRegister.py b/ampel/alert/reject/FullAlertRegister.py index c5e9db1..006d0d4 100755 --- a/ampel/alert/reject/FullAlertRegister.py +++ b/ampel/alert/reject/FullAlertRegister.py @@ -7,12 +7,13 @@ # Last Modified Date: 26.05.2020 # Last Modified By: valery brinnel -from time import time from struct import pack -from typing import Literal, BinaryIO, ClassVar, Generator -from ampel.util.register import reg_iter -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from time import time +from typing import BinaryIO, ClassVar, Generator, Literal + from ampel.alert.reject.BaseAlertRegister import BaseAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from ampel.util.register import reg_iter class FullAlertRegister(BaseAlertRegister): diff --git a/ampel/alert/reject/GeneralActiveAlertRegister.py b/ampel/alert/reject/GeneralActiveAlertRegister.py index aec238e..380b8ca 100755 --- a/ampel/alert/reject/GeneralActiveAlertRegister.py +++ b/ampel/alert/reject/GeneralActiveAlertRegister.py @@ -7,11 +7,12 @@ # Last Modified Date: 24.11.2021 # Last Modified By: valery brinnel +from collections.abc import Sequence from struct import pack from typing import ClassVar -from collections.abc import Sequence -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol + from ampel.alert.reject.GeneralAlertRegister import GeneralAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class GeneralActiveAlertRegister(GeneralAlertRegister): diff --git a/ampel/alert/reject/GeneralAlertRegister.py b/ampel/alert/reject/GeneralAlertRegister.py index 91b0ca1..563c153 100755 --- a/ampel/alert/reject/GeneralAlertRegister.py +++ b/ampel/alert/reject/GeneralAlertRegister.py @@ -8,9 +8,10 @@ # Last Modified By: valery brinnel from struct import pack -from typing import Literal, BinaryIO, ClassVar -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from typing import BinaryIO, ClassVar, Literal + from ampel.alert.reject.BaseAlertRegister import BaseAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class GeneralAlertRegister(BaseAlertRegister): diff --git a/ampel/alert/reject/MinimalActiveAlertRegister.py b/ampel/alert/reject/MinimalActiveAlertRegister.py index 2be5312..f7297e0 100755 --- a/ampel/alert/reject/MinimalActiveAlertRegister.py +++ b/ampel/alert/reject/MinimalActiveAlertRegister.py @@ -7,11 +7,12 @@ # Last Modified Date: 27.06.2022 # Last Modified By: valery brinnel +from collections.abc import Sequence from struct import pack from typing import ClassVar -from collections.abc import Sequence -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol + from ampel.alert.reject.MinimalAlertRegister import MinimalAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class MinimalActiveAlertRegister(MinimalAlertRegister): diff --git a/ampel/alert/reject/MinimalAlertRegister.py b/ampel/alert/reject/MinimalAlertRegister.py index ea07b52..be40fbc 100755 --- a/ampel/alert/reject/MinimalAlertRegister.py +++ b/ampel/alert/reject/MinimalAlertRegister.py @@ -8,10 +8,11 @@ # Last Modified By: valery brinnel from struct import pack -from typing import ClassVar, Literal, Generator, BinaryIO -from ampel.util.register import reg_iter -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from typing import BinaryIO, ClassVar, Generator, Literal + from ampel.alert.reject.BaseAlertRegister import BaseAlertRegister +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from ampel.util.register import reg_iter class MinimalAlertRegister(BaseAlertRegister): diff --git a/ampel/dev/AutoCompleteBenchmark.py b/ampel/dev/AutoCompleteBenchmark.py index 0d0ac65..1a7f449 100755 --- a/ampel/dev/AutoCompleteBenchmark.py +++ b/ampel/dev/AutoCompleteBenchmark.py @@ -7,14 +7,16 @@ # Last Modified Date: 29.04.2020 # Last Modified By: valery brinnel -from time import time +from collections.abc import Callable, Sequence from functools import wraps -from pymongo import MongoClient from multiprocessing import Pool, Semaphore, shared_memory +from time import time from typing import Any -from collections.abc import Sequence, Callable -from ampel.types import ChannelId, StockId + +from pymongo import MongoClient + from ampel.core.AmpelContext import AmpelContext +from ampel.types import ChannelId, StockId def timeit(f): diff --git a/ampel/dev/UnitTestAlertSupplier.py b/ampel/dev/UnitTestAlertSupplier.py index cfa4824..9aa601b 100755 --- a/ampel/dev/UnitTestAlertSupplier.py +++ b/ampel/dev/UnitTestAlertSupplier.py @@ -7,8 +7,8 @@ # Last Modified Date: 24.11.2021 # Last Modified By: valery brinnel -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol from ampel.abstract.AbsAlertSupplier import AbsAlertSupplier +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class UnitTestAlertSupplier(AbsAlertSupplier): diff --git a/ampel/model/AlertConsumerModel.py b/ampel/model/AlertConsumerModel.py index 9381795..b8b4cb4 100755 --- a/ampel/model/AlertConsumerModel.py +++ b/ampel/model/AlertConsumerModel.py @@ -8,11 +8,12 @@ # Last Modified By: valery brinnel from collections.abc import Sequence -from ampel.model.UnitModel import UnitModel -from ampel.model.ingest.IngestDirective import IngestDirective -from ampel.model.ingest.DualIngestDirective import DualIngestDirective -from ampel.model.ingest.CompilerOptions import CompilerOptions + from ampel.base.AmpelBaseModel import AmpelBaseModel +from ampel.model.ingest.CompilerOptions import CompilerOptions +from ampel.model.ingest.DualIngestDirective import DualIngestDirective +from ampel.model.ingest.IngestDirective import IngestDirective +from ampel.model.UnitModel import UnitModel class AlertConsumerModel(AmpelBaseModel): diff --git a/ampel/template/AbsEasyChannelTemplate.py b/ampel/template/AbsEasyChannelTemplate.py index 2664dad..e26cced 100755 --- a/ampel/template/AbsEasyChannelTemplate.py +++ b/ampel/template/AbsEasyChannelTemplate.py @@ -7,16 +7,18 @@ # Last Modified Date: 05.04.2023 # Last Modified By: valery brinnel -import ujson from typing import Any -from ampel.types import ChannelId + +import ujson + +from ampel.abstract.AbsChannelTemplate import AbsChannelTemplate +from ampel.config.builder.FirstPassConfig import FirstPassConfig from ampel.log.AmpelLogger import AmpelLogger from ampel.model.ChannelModel import ChannelModel -from ampel.model.ingest.T2Compute import T2Compute from ampel.model.ingest.FilterModel import FilterModel -from ampel.config.builder.FirstPassConfig import FirstPassConfig -from ampel.abstract.AbsChannelTemplate import AbsChannelTemplate -from ampel.util.template import filter_units, resolve_shortcut, check_tied_units +from ampel.model.ingest.T2Compute import T2Compute +from ampel.types import ChannelId +from ampel.util.template import check_tied_units, filter_units, resolve_shortcut class AbsEasyChannelTemplate(AbsChannelTemplate, abstract=True): diff --git a/setup.py b/setup.py index 1565800..e11e301 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ # Last Modified Date: 30.03.2023 # Last Modified By: valery brinnel -from setuptools import setup, find_namespace_packages +from setuptools import find_namespace_packages, setup package_data = { 'conf': [ diff --git a/tests/conftest.py b/tests/conftest.py index 96b8677..f74ac72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ -from ampel.abstract.AbsAlertFilter import AbsAlertFilter -from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +from pathlib import Path + import mongomock import pytest import yaml -from pathlib import Path +from ampel.abstract.AbsAlertFilter import AbsAlertFilter from ampel.dev.DevAmpelContext import DevAmpelContext -from ampel.test.dummy import DummyPointT2Unit, DummyStateT2Unit, DummyStockT2Unit +from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol @pytest.fixture diff --git a/tests/test_AbsAlertLoader.py b/tests/test_AbsAlertLoader.py index 82f190f..668e869 100644 --- a/tests/test_AbsAlertLoader.py +++ b/tests/test_AbsAlertLoader.py @@ -1,14 +1,15 @@ -import pytest import tarfile import uuid from io import BytesIO from pathlib import Path +import pytest + from ampel.abstract.AbsAlertLoader import AbsAlertLoader from ampel.alert.load.DirAlertLoader import DirAlertLoader +from ampel.alert.load.DirFileNamesLoader import DirFileNamesLoader from ampel.alert.load.FileAlertLoader import FileAlertLoader from ampel.alert.load.TarAlertLoader import TarAlertLoader -from ampel.alert.load.DirFileNamesLoader import DirFileNamesLoader def test_dummy(): diff --git a/tests/test_AbsEasyChannelTemplate.py b/tests/test_AbsEasyChannelTemplate.py index c9cf520..e4f5dd6 100644 --- a/tests/test_AbsEasyChannelTemplate.py +++ b/tests/test_AbsEasyChannelTemplate.py @@ -1,9 +1,10 @@ -from typing import Any, TYPE_CHECKING -from ampel.model.ChannelModel import ChannelModel -import pytest, yaml import contextlib +from typing import TYPE_CHECKING, Any + +import pytest from ampel.log.AmpelLogger import AmpelLogger +from ampel.model.ChannelModel import ChannelModel from ampel.template.AbsEasyChannelTemplate import AbsEasyChannelTemplate if TYPE_CHECKING: diff --git a/tests/test_AlertConsumer.py b/tests/test_AlertConsumer.py index 0552a89..ceac884 100644 --- a/tests/test_AlertConsumer.py +++ b/tests/test_AlertConsumer.py @@ -7,23 +7,26 @@ # Last Modified Date: 24.11.2021 # Last Modified By: vb -import pytest -import os, signal, time, threading +import os +import signal +import threading +import time from contextlib import contextmanager -from ampel.dev.DevAmpelContext import DevAmpelContext -from ampel.model.ingest.IngestBody import IngestBody -from ampel.model.ingest.IngestDirective import IngestDirective -from ampel.model.ingest.T1Combine import T1Combine -from ampel.model.ingest.T2Compute import T2Compute +import pytest from ampel.alert.AlertConsumer import AlertConsumer from ampel.alert.AlertConsumerError import AlertConsumerError from ampel.alert.AmpelAlert import AmpelAlert from ampel.alert.filter.BasicMultiFilter import BasicMultiFilter +from ampel.dev.DevAmpelContext import DevAmpelContext from ampel.dev.UnitTestAlertSupplier import UnitTestAlertSupplier from ampel.metrics.AmpelMetricsRegistry import AmpelMetricsRegistry from ampel.model.ingest.FilterModel import FilterModel +from ampel.model.ingest.IngestBody import IngestBody +from ampel.model.ingest.IngestDirective import IngestDirective +from ampel.model.ingest.T1Combine import T1Combine +from ampel.model.ingest.T2Compute import T2Compute @contextmanager diff --git a/tests/test_EasyAlertConsumerTemplate.py b/tests/test_EasyAlertConsumerTemplate.py index 78d9467..402ee1b 100644 --- a/tests/test_EasyAlertConsumerTemplate.py +++ b/tests/test_EasyAlertConsumerTemplate.py @@ -1,22 +1,21 @@ +import sys from contextlib import contextmanager from pathlib import Path -import sys from typing import TYPE_CHECKING -from ampel.alert.AlertConsumer import AlertConsumer import pytest +from pytest_mock import MockerFixture +from ampel.alert.AlertConsumer import AlertConsumer +from ampel.alert.AmpelAlert import AmpelAlert +from ampel.cli.main import main from ampel.dev.DevAmpelContext import DevAmpelContext from ampel.log.AmpelLogger import AmpelLogger -from ampel.template.EasyAlertConsumerTemplate import EasyAlertConsumerTemplate -from ampel.alert.AmpelAlert import AmpelAlert from ampel.model.UnitModel import UnitModel - -from ampel.cli.main import main -from pytest_mock import MockerFixture +from ampel.template.EasyAlertConsumerTemplate import EasyAlertConsumerTemplate if TYPE_CHECKING: - from ampel.config.builder.FirstPassConfig import FirstPassConfig + pass @pytest.mark.parametrize(["muxer"], [(None,), ("DummyMuxer",)]) From a2afa674431fdedacfebb9152660ab1684be3422 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 20:09:31 +0100 Subject: [PATCH 2/8] ruff: basic cleanup --- ampel/abstract/AbsAlertSupplier.py | 2 +- ampel/alert/AlertConsumer.py | 38 +++++---- ampel/alert/FilterBlock.py | 88 ++++++++++----------- ampel/alert/FilteringAlertSupplier.py | 2 +- ampel/alert/filter/BasicMultiFilter.py | 2 +- ampel/alert/load/TarAlertLoader.py | 5 +- ampel/alert/reject/FullAlertRegister.py | 3 +- ampel/alert/reject/MinimalAlertRegister.py | 3 +- ampel/dev/AutoCompleteBenchmark.py | 12 +-- ampel/template/EasyAlertConsumerTemplate.py | 3 +- tests/conftest.py | 16 ++-- tests/test_AbsAlertLoader.py | 4 +- tests/test_AbsEasyChannelTemplate.py | 5 +- tests/test_AlertConsumer.py | 4 +- tests/test_EasyAlertConsumerTemplate.py | 8 +- 15 files changed, 99 insertions(+), 96 deletions(-) diff --git a/ampel/abstract/AbsAlertSupplier.py b/ampel/abstract/AbsAlertSupplier.py index 1b2e885..d56d9f4 100755 --- a/ampel/abstract/AbsAlertSupplier.py +++ b/ampel/abstract/AbsAlertSupplier.py @@ -7,7 +7,7 @@ # Last Modified Date: 19.12.2022 # Last Modified By: valery brinnel -from typing import Iterator +from collections.abc import Iterator from ampel.base.AmpelABC import AmpelABC from ampel.base.AmpelUnit import AmpelUnit diff --git a/ampel/alert/AlertConsumer.py b/ampel/alert/AlertConsumer.py index 96e5118..a252a55 100755 --- a/ampel/alert/AlertConsumer.py +++ b/ampel/alert/AlertConsumer.py @@ -10,7 +10,7 @@ import sys from collections.abc import Sequence from signal import SIGINT, SIGTERM, default_int_handler, signal -from typing import Any +from typing import TYPE_CHECKING, Any from pymongo.errors import PyMongoError from typing_extensions import Self @@ -36,6 +36,9 @@ from ampel.util.freeze import recursive_unfreeze from ampel.util.mappings import get_by_path, merge_dict +if TYPE_CHECKING: + from ampel.alert.FilterBlock import FilterBlock + from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol class AlertConsumer(AbsEventUnit, AlertConsumerModel): """ @@ -258,17 +261,19 @@ def proceed(self, event_hdlr: EventHandler) -> int: self._fbh.ready(logger, run_id) # Shortcuts - report_filter_error = lambda e, alert, fblock: self._report_ap_error( - e, event_hdlr, logger, - extra = {'a': alert.id, 'section': 'filter', 'c': fblock.channel} - ) + def report_filter_error(e: Exception, alert: "AmpelAlertProtocol", fblock: "FilterBlock"): + self._report_ap_error( + e, event_hdlr, logger, + extra = {'a': alert.id, 'section': 'filter', 'c': fblock.channel} + ) - report_ingest_error = lambda e, alert, filter_results: self._report_ap_error( - e, event_hdlr, logger, extra={ - 'a': alert.id, 'section': 'ingest', - 'c': [self.directives[el[0]].channel for el in filter_results] - } - ) + def report_ingest_error(e: Exception, alert: "AmpelAlertProtocol", filter_results: Sequence[tuple[int, bool|int]]): + self._report_ap_error( + e, event_hdlr, logger, extra={ + 'a': alert.id, 'section': 'ingest', + 'c': [self.directives[el[0]].channel for el in filter_results] + } + ) # Process alerts ################ @@ -321,12 +326,11 @@ def proceed(self, event_hdlr: EventHandler) -> int: if self.raise_exc: raise e - else: - if self.error_max: - err += 1 - if err == self.error_max: - logger.error("Max number of error reached, breaking alert processing") - self.set_cancel_run(AlertConsumerError.TOO_MANY_ERRORS) + if self.error_max: + err += 1 + if err == self.error_max: + logger.error("Max number of error reached, breaking alert processing") + self.set_cancel_run(AlertConsumerError.TOO_MANY_ERRORS) else: # if bypassing filters, track passing rates at top level for counter in stats.filter_accepted: diff --git a/ampel/alert/FilterBlock.py b/ampel/alert/FilterBlock.py index 812bcb7..b6762e9 100755 --- a/ampel/alert/FilterBlock.py +++ b/ampel/alert/FilterBlock.py @@ -182,67 +182,63 @@ def filter(self, alert: AmpelAlertProtocol) -> tuple[int, int | bool | None]: return self.idx, res # Filter rejected alert - else: - - self._stat_rejected.inc() + self._stat_rejected.inc() - # 'overrule' or 'silent_overrule' requested for this filter - if self.overrule and alert.stock in self.stock_ids: + # 'overrule' or 'silent_overrule' requested for this filter + if self.overrule and alert.stock in self.stock_ids: - extra_ac = {'a': alert.id, 'ac': True, 's': alert.stock, 'c': self.channel} + extra_ac = {'a': alert.id, 'ac': True, 's': alert.stock, 'c': self.channel} - # Main logger feedback - self.log(INFO, None, extra=extra_ac) + # Main logger feedback + self.log(INFO, None, extra=extra_ac) - # Update count - self._stat_autocomplete.inc() + # Update count + self._stat_autocomplete.inc() - # Rejected alerts notifications can go to rejected log collection - # even though it was "auto-completed" because it - # was actually rejected by the filter/channel - if self.update_rej: + # Rejected alerts notifications can go to rejected log collection + # even though it was "auto-completed" because it + # was actually rejected by the filter/channel + if self.update_rej: - if self.buffer: - if self.rej_log_handler: - # Clears the buffer - self.forward(self.rej_log_handler, stock=alert.stock, extra=extra_ac) - else: - self.buffer.clear() - - # Log minimal entry if channel did not log anything + if self.buffer: + if self.rej_log_handler: + # Clears the buffer + self.forward(self.rej_log_handler, stock=alert.stock, extra=extra_ac) else: - if self.rej_log_handle: - lrec = LightLogRecord(0, 0, None) - lrec.stock = alert.stock - lrec.extra = extra_ac - self.rej_log_handle(lrec) + self.buffer.clear() - if self.file: - self.file(alert, res) + # Log minimal entry if channel did not log anything + else: + if self.rej_log_handle: + lrec = LightLogRecord(0, 0, None) + lrec.stock = alert.stock + lrec.extra = extra_ac + self.rej_log_handle(lrec) - # Use default t2 units (no group) as filter results - return self.overrule + if self.file: + self.file(alert, res) - else: + # Use default t2 units (no group) as filter results + return self.overrule - if self.buffer: + if self.buffer: - # Save possibly existing error to 'main' logs - if self.buf_hdlr.has_error: - self.forward( - self.logger, stock=alert.stock, extra={'a': alert.id}, - clear=not self.rej_log_handler - ) + # Save possibly existing error to 'main' logs + if self.buf_hdlr.has_error: + self.forward( + self.logger, stock=alert.stock, extra={'a': alert.id}, + clear=not self.rej_log_handler + ) - if self.rej_log_handler: - # Send rejected logs to dedicated separate logger/handler - self.forward(self.rej_log_handler, stock=alert.stock, extra={'a': alert.id}) + if self.rej_log_handler: + # Send rejected logs to dedicated separate logger/handler + self.forward(self.rej_log_handler, stock=alert.stock, extra={'a': alert.id}) - if self.file: - self.file(alert, res) + if self.file: + self.file(alert, res) - # return rejection result - return self.rej + # return rejection result + return self.rej def ready(self, logger: AmpelLogger, run_id: int) -> None: diff --git a/ampel/alert/FilteringAlertSupplier.py b/ampel/alert/FilteringAlertSupplier.py index 352f307..4783a85 100755 --- a/ampel/alert/FilteringAlertSupplier.py +++ b/ampel/alert/FilteringAlertSupplier.py @@ -7,7 +7,7 @@ # Last Modified Date: 24.11.2021 # Last Modified By: valery brinnel -from typing import Iterator +from collections.abc import Iterator from ampel.abstract.AbsAlertSupplier import AbsAlertSupplier from ampel.base.AuxUnitRegister import AuxUnitRegister diff --git a/ampel/alert/filter/BasicMultiFilter.py b/ampel/alert/filter/BasicMultiFilter.py index 9f69c06..f0e93a1 100755 --- a/ampel/alert/filter/BasicMultiFilter.py +++ b/ampel/alert/filter/BasicMultiFilter.py @@ -122,7 +122,7 @@ def process(self, alert: AmpelAlertProtocol) -> bool: current_res = False - for i, param in enumerate(filter_res): + for i, _ in enumerate(filter_res): if i == 0: current_res = filter_res[i] diff --git a/ampel/alert/load/TarAlertLoader.py b/ampel/alert/load/TarAlertLoader.py index e43baf6..491187a 100755 --- a/ampel/alert/load/TarAlertLoader.py +++ b/ampel/alert/load/TarAlertLoader.py @@ -108,9 +108,8 @@ def __next__(self) -> IOBase: self._chained_tal = TarAlertLoader(file_obj=file_obj) if (subfile_obj := self.get_chained_next()) is not None: return subfile_obj - else: - return next(self) - elif tar_info.name.endswith('.gz'): + return next(self) + if tar_info.name.endswith('.gz'): return GzipFile(mode="rb", fileobj=file_obj) # type: ignore[return-value] return file_obj diff --git a/ampel/alert/reject/FullAlertRegister.py b/ampel/alert/reject/FullAlertRegister.py index 006d0d4..68fdefa 100755 --- a/ampel/alert/reject/FullAlertRegister.py +++ b/ampel/alert/reject/FullAlertRegister.py @@ -7,9 +7,10 @@ # Last Modified Date: 26.05.2020 # Last Modified By: valery brinnel +from collections.abc import Generator from struct import pack from time import time -from typing import BinaryIO, ClassVar, Generator, Literal +from typing import BinaryIO, ClassVar, Literal from ampel.alert.reject.BaseAlertRegister import BaseAlertRegister from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol diff --git a/ampel/alert/reject/MinimalAlertRegister.py b/ampel/alert/reject/MinimalAlertRegister.py index be40fbc..2fa94a6 100755 --- a/ampel/alert/reject/MinimalAlertRegister.py +++ b/ampel/alert/reject/MinimalAlertRegister.py @@ -7,8 +7,9 @@ # Last Modified Date: 27.06.2022 # Last Modified By: valery brinnel +from collections.abc import Generator from struct import pack -from typing import BinaryIO, ClassVar, Generator, Literal +from typing import BinaryIO, ClassVar, Literal from ampel.alert.reject.BaseAlertRegister import BaseAlertRegister from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol diff --git a/ampel/dev/AutoCompleteBenchmark.py b/ampel/dev/AutoCompleteBenchmark.py index 1a7f449..509e0a6 100755 --- a/ampel/dev/AutoCompleteBenchmark.py +++ b/ampel/dev/AutoCompleteBenchmark.py @@ -188,7 +188,7 @@ def benchmark(self, channel: ChannelId): @timeit def get_ids_using_find(self, channel: ChannelId, *, verbose=True): """ Warning: slow for large collections """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): return {el['_id'] for el in self._stock_col.find({'channel': channel}, {'_id': 1})} return {k: self.get_ids_using_find(k, verbose=verbose) for k in channel} @@ -196,7 +196,7 @@ def get_ids_using_find(self, channel: ChannelId, *, verbose=True): @timeit def get_ids_using_parallel_find(self, channel: ChannelId, *, batch_size=1000000, verbose=True): """ Winner method """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): return {el['_id'] for el in self._stock_col.find({'channel': channel}, {'_id': 1}).batch_size(batch_size)} pool = Pool(4) @@ -222,7 +222,7 @@ def get_ids_using_parallel_find(self, channel: ChannelId, *, batch_size=1000000, @timeit def get_ids_using_distinct(self, channel: ChannelId, *, verbose: bool = True): """ Warning: fails for large collections """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): return set(self._stock_col.distinct('_id', filter={'channel': channel})) return {k: self.get_ids_using_distinct(k, verbose=verbose) for k in channel} @@ -230,7 +230,7 @@ def get_ids_using_distinct(self, channel: ChannelId, *, verbose: bool = True): @timeit def get_ids_using_aggregate(self, channel: ChannelId, *, verbose: bool = True): """ Warning: fails for large collections """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): return next( self._stock_col.aggregate( [ @@ -252,7 +252,7 @@ def get_ids_using_paged_aggregate(self, channel: ChannelId, *, verbose: bool = T and result in noticible performance drawbacks. All in all, parallel "parallel find(...)" works all the time better in all circumstances. """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): s = set() skip = 0 @@ -286,7 +286,7 @@ def get_ids_using_paged_group_aggregate(self, channel: ChannelId, *, verbose: bo Firing one request per channel and grouping the result in python yield much better performance. """ - if isinstance(channel, (int, str)): + if isinstance(channel, int | str): return self.get_ids_using_paged_aggregate(channel) skip = 0 diff --git a/ampel/template/EasyAlertConsumerTemplate.py b/ampel/template/EasyAlertConsumerTemplate.py index 9301e29..4917d33 100644 --- a/ampel/template/EasyAlertConsumerTemplate.py +++ b/ampel/template/EasyAlertConsumerTemplate.py @@ -71,8 +71,7 @@ def _config_as_dict(arg: str | UnitModel) -> dict[str, Any]: def _config_as_dict(arg: None | str | UnitModel) -> None | dict[str, Any]: if arg is None: return None - else: - return (arg if isinstance(arg, UnitModel) else UnitModel(unit=arg)).dict(exclude_unset=True) + return (arg if isinstance(arg, UnitModel) else UnitModel(unit=arg)).dict(exclude_unset=True) def _get_supplier(self) -> dict[str, Any]: diff --git a/tests/conftest.py b/tests/conftest.py index f74ac72..d69918d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,24 +9,24 @@ from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol -@pytest.fixture -def patch_mongo(monkeypatch): +@pytest.fixture() +def _patch_mongo(monkeypatch): monkeypatch.setattr("ampel.core.AmpelDB.MongoClient", mongomock.MongoClient) -@pytest.fixture +@pytest.fixture() def testing_config(): return Path(__file__).parent / "testing-config.yaml" -@pytest.fixture +@pytest.fixture() def first_pass_config(testing_config): with open(testing_config, "rb") as f: return yaml.safe_load(f) -@pytest.fixture -def dev_context(patch_mongo, testing_config): +@pytest.fixture() +def dev_context(_patch_mongo, testing_config): return DevAmpelContext.load(testing_config) @@ -35,8 +35,8 @@ def process(self, alert: AmpelAlertProtocol) -> None | bool | int: return True -@pytest.fixture -def dummy_units(dev_context: DevAmpelContext): +@pytest.fixture() +def _dummy_units(dev_context: DevAmpelContext): # register dummy units in-process so gen_config_id can find them from ampel.test import dummy diff --git a/tests/test_AbsAlertLoader.py b/tests/test_AbsAlertLoader.py index 668e869..32b06da 100644 --- a/tests/test_AbsAlertLoader.py +++ b/tests/test_AbsAlertLoader.py @@ -57,11 +57,11 @@ def test_FileAlertLoader(dummy_alert: tuple[Path, bytes]): for item in loader: assert item.read() == content - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Parameter 'files' cannot be empty"): FileAlertLoader(files=[]) -@pytest.mark.parametrize("klass", (DirAlertLoader, DirFileNamesLoader)) +@pytest.mark.parametrize("klass", [DirAlertLoader, DirFileNamesLoader]) def test_DirAlertLoader(klass, dummy_alert: tuple[Path, bytes]): path, content = dummy_alert loader = klass( diff --git a/tests/test_AbsEasyChannelTemplate.py b/tests/test_AbsEasyChannelTemplate.py index e4f5dd6..8ffcaae 100644 --- a/tests/test_AbsEasyChannelTemplate.py +++ b/tests/test_AbsEasyChannelTemplate.py @@ -35,7 +35,7 @@ def get_processes( @pytest.mark.parametrize( - "t2_compute,target,expected,exception", + ("t2_compute","target","expected","exception"), [ # single, statebound T2 ( @@ -105,7 +105,8 @@ def get_processes( ), ], ) -def test_state_t2_instantiation(t2_compute, target, expected, exception, dev_context, dummy_units): +@pytest.mark.usefixtures("_dummy_units") +def test_state_t2_instantiation(t2_compute, target, expected, exception, dev_context): """ Template creates state T2s and checks for missing dependencies """ diff --git a/tests/test_AlertConsumer.py b/tests/test_AlertConsumer.py index ceac884..a6947c7 100644 --- a/tests/test_AlertConsumer.py +++ b/tests/test_AlertConsumer.py @@ -47,9 +47,9 @@ def collect_diff(store): store.update(delta) -@pytest.fixture +@pytest.fixture() def single_source_directive( - dev_context: DevAmpelContext, dummy_units + dev_context: DevAmpelContext, _dummy_units ) -> IngestDirective: return IngestDirective( diff --git a/tests/test_EasyAlertConsumerTemplate.py b/tests/test_EasyAlertConsumerTemplate.py index 402ee1b..936fcab 100644 --- a/tests/test_EasyAlertConsumerTemplate.py +++ b/tests/test_EasyAlertConsumerTemplate.py @@ -18,8 +18,9 @@ pass -@pytest.mark.parametrize(["muxer"], [(None,), ("DummyMuxer",)]) -def test_instantiation(dev_context: DevAmpelContext, muxer, dummy_units): +@pytest.mark.parametrize("muxer", [None, "DummyMuxer"]) +@pytest.mark.usefixtures("_dummy_units") +def test_instantiation(dev_context: DevAmpelContext, muxer): tpl = EasyAlertConsumerTemplate( **{ "channel": "TEST_CHANNEL", @@ -92,8 +93,9 @@ def run(args: list[str]) -> None | int | str: return se.code +@pytest.mark.usefixtures("_dummy_units") def test_job_file( - testing_config, dev_context: DevAmpelContext, dummy_units, mocker: MockerFixture + testing_config, dev_context: DevAmpelContext, mocker: MockerFixture ): mock = mocker.patch.object( AlertConsumer, "proceed", side_effect=AlertConsumer.proceed, autospec=True From d87ec10873ebabc3d19c8812879cc61de41b8d13 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 20:12:06 +0100 Subject: [PATCH 3/8] ruff: noqa "private" member access --- ampel/alert/FilterBlocksHandler.py | 4 ++-- ampel/alert/filter/BasicMultiFilter.py | 1 + ampel/dev/AutoCompleteBenchmark.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ampel/alert/FilterBlocksHandler.py b/ampel/alert/FilterBlocksHandler.py index b33d4ad..dda609f 100755 --- a/ampel/alert/FilterBlocksHandler.py +++ b/ampel/alert/FilterBlocksHandler.py @@ -106,13 +106,13 @@ def __init__(self, # Note: channel names can be integers self.chan_names = [ f"{fb.channel}" for fb in self.filter_blocks - if fb.channel in context.config._config['channel'] + if fb.channel in context.config._config['channel'] # noqa: SLF001 ] # Check that channels defined in directives exist in ampel config if len(self.chan_names) != len(self.filter_blocks): for fb in self.filter_blocks: - if fb.channel not in context.config._config['channel']: + if fb.channel not in context.config._config['channel']: # noqa: SLF001 raise ValueError(f"Channel {fb.channel} unknown in ampel config") if len(self.filter_blocks) == 1 and db_log_format == "compact": diff --git a/ampel/alert/filter/BasicMultiFilter.py b/ampel/alert/filter/BasicMultiFilter.py index f0e93a1..f0fbc0f 100755 --- a/ampel/alert/filter/BasicMultiFilter.py +++ b/ampel/alert/filter/BasicMultiFilter.py @@ -15,6 +15,7 @@ from ampel.base.AmpelBaseModel import AmpelBaseModel from ampel.protocol.AmpelAlertProtocol import AmpelAlertProtocol +# ruff: noqa: SLF001 class PhotoAlertQuery(AmpelBaseModel): """ diff --git a/ampel/dev/AutoCompleteBenchmark.py b/ampel/dev/AutoCompleteBenchmark.py index 509e0a6..8a536fe 100755 --- a/ampel/dev/AutoCompleteBenchmark.py +++ b/ampel/dev/AutoCompleteBenchmark.py @@ -347,7 +347,7 @@ def get_ids_using_parallel_aggregate(self, start = time() skip = 0 pool = Pool(4) - self.sem = Semaphore(pool._processes) # type: ignore + self.sem = Semaphore(pool._processes) # type: ignore # noqa: SLF001 self.keys = list(channels) results = [] From 5331887b763fef65aea0c68df26af2c63144aadf Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 20:16:37 +0100 Subject: [PATCH 4/8] ruff: simplify --- ampel/alert/AlertConsumer.py | 9 ++++----- ampel/alert/load/TarballWalker.py | 20 +++++++++----------- ampel/alert/reject/BaseAlertRegister.py | 9 ++++++--- ampel/alert/reject/DBRejectedLogsHandler.py | 5 +---- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/ampel/alert/AlertConsumer.py b/ampel/alert/AlertConsumer.py index a252a55..c48675a 100755 --- a/ampel/alert/AlertConsumer.py +++ b/ampel/alert/AlertConsumer.py @@ -9,6 +9,7 @@ import sys from collections.abc import Sequence +from contextlib import suppress from signal import SIGINT, SIGTERM, default_int_handler, signal from typing import TYPE_CHECKING, Any @@ -468,8 +469,8 @@ def _report_ap_error( } if extra: - for k in extra.keys(): - info[k] = extra[k] + for k, v in extra.items(): + info[k] = v # Try to insert doc into trouble collection (raises no exception) # Possible exception will be logged out to console in any case @@ -479,10 +480,8 @@ def _report_ap_error( @staticmethod def print_feedback(arg: Any, suffix: str = "") -> None: print("") # ^C in console - try: + with suppress(Exception): arg = AlertConsumerError(arg) - except Exception: - pass s = f"[{arg.name if isinstance(arg, AlertConsumerError) else arg}] Interrupting run {suffix}" print("+" * len(s)) print(s) diff --git a/ampel/alert/load/TarballWalker.py b/ampel/alert/load/TarballWalker.py index e538ad8..7d07a20 100644 --- a/ampel/alert/load/TarballWalker.py +++ b/ampel/alert/load/TarballWalker.py @@ -24,19 +24,17 @@ def __init__(self, tarpath, start=0, stop=None): def get_files(self): - tar_file = open(self.tarpath, 'rb') - count = -1 + with open(self.tarpath, 'rb') as tar_file: + count = -1 - for fileobj in self._walk(tar_file): + for fileobj in self._walk(tar_file): - count += 1 - if count < self.start: - continue - if self.stop is not None and count > self.stop: - break - yield fileobj - - tar_file.close() + count += 1 + if count < self.start: + continue + if self.stop is not None and count > self.stop: + break + yield fileobj def _walk(self, fileobj): diff --git a/ampel/alert/reject/BaseAlertRegister.py b/ampel/alert/reject/BaseAlertRegister.py index a6b8c73..56ae70b 100755 --- a/ampel/alert/reject/BaseAlertRegister.py +++ b/ampel/alert/reject/BaseAlertRegister.py @@ -126,9 +126,12 @@ def check_rename(self, header: dict[str, Any]) -> bool: if not self.file_cap: return False - if 'runs' in self.file_cap: # type: ignore[operator] - if isinstance(header['run'], list) and len(header['run']) > self.file_cap['runs']: - return True + if ( + 'runs' in self.file_cap and # type: ignore[operator] + isinstance(header['run'], list) and + len(header['run']) > self.file_cap['runs'] + ): + return True return super().check_rename(header) diff --git a/ampel/alert/reject/DBRejectedLogsHandler.py b/ampel/alert/reject/DBRejectedLogsHandler.py index 5d4ba34..72e2cd3 100755 --- a/ampel/alert/reject/DBRejectedLogsHandler.py +++ b/ampel/alert/reject/DBRejectedLogsHandler.py @@ -98,10 +98,7 @@ def handle(self, record: LightLogRecord | LogRecord) -> None: # If duplication exists between keys in extra and in standard rec, # the corresponding extra items will be overwritten (and thus ignored) - if 'extra' in rd: - d = {k: rd['extra'][k] for k in rd['extra']} - else: - d = {} + d = {k: rd["extra"][k] for k in rd["extra"]} if "extra" in rd else {} # 'alert' and 'stock' must exist in the log record, # otherwise, the AP made a mistake From 53447d0543f49f123c759a61e37d16b164bd1c32 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 20:17:53 +0100 Subject: [PATCH 5/8] ruff: pylint --- ampel/alert/FilterBlock.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ampel/alert/FilterBlock.py b/ampel/alert/FilterBlock.py index b6762e9..89e02ba 100755 --- a/ampel/alert/FilterBlock.py +++ b/ampel/alert/FilterBlock.py @@ -34,6 +34,8 @@ def no_filter(alert: Any) -> bool: return True +# ruff: noqa: PLE0237 + class FilterBlock: """ Helper class for AlertConsumer. @@ -208,12 +210,11 @@ def filter(self, alert: AmpelAlertProtocol) -> tuple[int, int | bool | None]: self.buffer.clear() # Log minimal entry if channel did not log anything - else: - if self.rej_log_handle: - lrec = LightLogRecord(0, 0, None) - lrec.stock = alert.stock - lrec.extra = extra_ac - self.rej_log_handle(lrec) + elif self.rej_log_handle: + lrec = LightLogRecord(0, 0, None) + lrec.stock = alert.stock + lrec.extra = extra_ac + self.rej_log_handle(lrec) if self.file: self.file(alert, res) From df85cc48f9e8e1c01ffe0759f95f7e6adb9a1333 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 20:48:07 +0100 Subject: [PATCH 6/8] ruff: perf --- ampel/alert/AlertConsumer.py | 2 +- ampel/alert/AlertFileList.py | 18 ++++++++---------- ampel/alert/filter/BasicMultiFilter.py | 20 ++++++++------------ ampel/alert/load/DirAlertLoader.py | 18 ++++++++---------- ampel/dev/AutoCompleteBenchmark.py | 2 +- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/ampel/alert/AlertConsumer.py b/ampel/alert/AlertConsumer.py index c48675a..50faecd 100755 --- a/ampel/alert/AlertConsumer.py +++ b/ampel/alert/AlertConsumer.py @@ -312,7 +312,7 @@ def report_ingest_error(e: Exception, alert: "AmpelAlertProtocol", filter_result filter_results.append(res) # type: ignore[arg-type] # Unrecoverable (logging related) errors - except (PyMongoError, AmpelLoggingError) as e: + except (PyMongoError, AmpelLoggingError) as e: # noqa: PERF203 print("%s: abording run() procedure" % e.__class__.__name__) report_filter_error(e, alert, fblock) raise e diff --git a/ampel/alert/AlertFileList.py b/ampel/alert/AlertFileList.py index 7e291a9..ac1195b 100644 --- a/ampel/alert/AlertFileList.py +++ b/ampel/alert/AlertFileList.py @@ -66,19 +66,17 @@ def build_file_list(self): if self.min_index is not None: self.logger.debug("Filtering files using min_index criterium") - out_files = [] - for f in all_files: - if int(os.path.basename(f).split(".")[0]) >= self.min_index: - out_files.append(f) - all_files = out_files + all_files = [ + f for f in all_files + if int(os.path.basename(f).split(".")[0]) >= self.min_index + ] if self.max_index is not None: self.logger.debug("Filtering files using max_index criterium") - out_files = [] - for f in all_files: - if int(os.path.basename(f).split(".")[0]) <= self.max_index: - out_files.append(f) - all_files = out_files + all_files = [ + f for f in all_files + if int(os.path.basename(f).split(".")[0]) <= self.max_index + ] if self.max_entries is not None: self.logger.debug("Filtering files using max_entries criterium") diff --git a/ampel/alert/filter/BasicMultiFilter.py b/ampel/alert/filter/BasicMultiFilter.py index f0fbc0f..19f17ec 100755 --- a/ampel/alert/filter/BasicMultiFilter.py +++ b/ampel/alert/filter/BasicMultiFilter.py @@ -108,18 +108,14 @@ def process(self, alert: AmpelAlertProtocol) -> bool: ] """ - filter_res = [] - - for param in self.filters: - - filter_res.append( - param._operator( - len( - alert.get_values('candid', filters = param._criteria) - ), - param.len - ) - ) + filter_res = [ + param._operator( + len( + alert.get_values('candid', filters = param._criteria) + ), + param.len + ) for param in self.filters + ] current_res = False diff --git a/ampel/alert/load/DirAlertLoader.py b/ampel/alert/load/DirAlertLoader.py index c4b8fef..fc7f275 100755 --- a/ampel/alert/load/DirAlertLoader.py +++ b/ampel/alert/load/DirAlertLoader.py @@ -70,19 +70,17 @@ def build_file_list(self) -> None: if self.min_index is not None: self.logger.debug("Filtering files using min_index criterium") - out_files = [] - for f in all_files: - if int(os.path.basename(f).split(".")[0]) >= self.min_index: - out_files.append(f) - all_files = out_files + all_files = [ + f for f in all_files + if int(os.path.basename(f).split(".")[0]) >= self.min_index + ] if self.max_index is not None: self.logger.debug("Filtering files using max_index criterium") - out_files = [] - for f in all_files: - if int(os.path.basename(f).split(".")[0]) <= self.max_index: - out_files.append(f) - all_files = out_files + all_files = [ + f for f in all_files + if int(os.path.basename(f).split(".")[0]) <= self.max_index + ] if self.max_entries is not None: self.logger.debug("Filtering files using max_entries criterium") diff --git a/ampel/dev/AutoCompleteBenchmark.py b/ampel/dev/AutoCompleteBenchmark.py index 8a536fe..f67fd76 100755 --- a/ampel/dev/AutoCompleteBenchmark.py +++ b/ampel/dev/AutoCompleteBenchmark.py @@ -178,7 +178,7 @@ def benchmark(self, channel: ChannelId): print(f'channel: {len(r)}') print('') print('') - except Exception as e: + except Exception as e: # noqa: PERF203 print(e) print(f'{name}(...) failed') print('') From cbef9ef206f4edd187cb71b9e6a1f5abd3673989 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 21:00:12 +0100 Subject: [PATCH 7/8] ruff: ruff-specific rules --- ampel/alert/AlertFileList.py | 2 +- ampel/alert/BaseAlertSupplier.py | 2 +- pyproject.toml | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ampel/alert/AlertFileList.py b/ampel/alert/AlertFileList.py index ac1195b..9e4c3f4 100644 --- a/ampel/alert/AlertFileList.py +++ b/ampel/alert/AlertFileList.py @@ -41,7 +41,7 @@ def set_index_range(self, min_index=None, max_index=None): self.min_index = min_index self.max_index = max_index self.logger.debug(f"Min index set to: {self.min_index}") - self.logger.debug(f"Max index set to: {str(self.max_index)}") + self.logger.debug(f"Max index set to: {self.max_index:s}") def set_max_entries(self, max_entries): diff --git a/ampel/alert/BaseAlertSupplier.py b/ampel/alert/BaseAlertSupplier.py index 4bba6a8..775a2fb 100755 --- a/ampel/alert/BaseAlertSupplier.py +++ b/ampel/alert/BaseAlertSupplier.py @@ -74,7 +74,7 @@ def __init__(self, **kwargs) -> None: elif self.deserialize == "avro": from fastavro import reader - def avro_next(arg: IOBase): # noqa: E306 + def avro_next(arg: IOBase): return next(reader(arg)) self._deserialize = avro_next diff --git a/pyproject.toml b/pyproject.toml index badc112..609e665 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,3 +80,37 @@ module = [ "fastavro", ] ignore_missing_imports = true + +[tool.ruff] +target-version = "py310" + +[tool.ruff.lint] +select = [ + "E4", + "E7", + "E9", + "F", + "I", + "UP", + "B", + "DTZ", + "T20", + "PT", + "RET", + "SLF", + "SIM", + "PL", + "PERF", + "RUF", +] +ignore = [ + "UP009", # UTF-8 encoding declaration is unnecessary + "PLR09", # too many (arguments|branches) + "PLR2004", # Magic value used in comparison + "RUF012", # mutable class properties (are harmless everywhere BaseModel is used) +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["T20", "E731", "SLF001", "PLR2004"] +"ampel/dev/AutoCompleteBenchmark.py" = ["T201"] +"ampel/alert/AlertConsumer.py" = ["T201"] From 2ecdea29acc31043b9adba3c483dfbdcccc8cae3 Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Tue, 30 Jan 2024 21:01:41 +0100 Subject: [PATCH 8/8] chore(ci): enable linting --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c99ce2..04d18ab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,5 +19,8 @@ jobs: uses: AmpelProject/Ampel-interface/.github/workflows/ci.yml@ci-py12-v3 secrets: inherit with: + lint: true # renovate: datasource=conda depName=conda-forge/python python-version: "3.12.1" + # renovate: datasource=pypi depName=ruff + ruff-version: "0.1.15"