diff --git a/docs/implementation/logging.md b/docs/implementation/logging.md index da20a5780..530e886f8 100644 --- a/docs/implementation/logging.md +++ b/docs/implementation/logging.md @@ -77,3 +77,21 @@ This will send metrics to a `metrics.log`: 2022-09-29 00:48:53,302 INFO METRIC: {"metric_type": "timer", "metric": "sync_duration", "value": 0.5258760452270508, "tags": {"stream": "countries", "context": {}, "status": "succeeded"}} 2022-09-29 00:48:53,303 INFO METRIC: {"metric_type": "counter", "metric": "record_count", "value": 250, "tags": {"stream": "countries", "context": {}}} ``` + +## For package developers + +If you're developing a tap or target package, you can put a `default_loggging.yml` file in the package root to set the default logging configuration for your package. This file will be used if the `SINGER_SDK_LOG_CONFIG` environment variable is not set: + +``` +. +├── README.md +├── poetry.lock +├── pyproject.toml +└── tap_example +    ├── __init__.py +    ├── __main__.py +    ├── default_logging.yml # <-- This file will be used if SINGER_SDK_LOG_CONFIG is not set +    ├── client.py +    ├── streams.py +    └── tap.py +``` diff --git a/singer_sdk/logger.py b/singer_sdk/logger.py deleted file mode 100644 index 31a733f5c..000000000 --- a/singer_sdk/logger.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Logging configuration for the Singer SDK.""" - -from __future__ import annotations - -import logging -import os -import typing as t -from pathlib import Path - -import yaml - -from singer_sdk.metrics import METRICS_LOG_LEVEL_SETTING, METRICS_LOGGER_NAME - -if t.TYPE_CHECKING: - from singer_sdk.helpers._compat import Traversable - -__all__ = ["setup_logging"] - - -def load_yaml_logging_config(path: Traversable | Path) -> t.Any: # noqa: ANN401 - """Load the logging config from the YAML file. - - Args: - path: A path to the YAML file. - - Returns: - The logging config. - """ - with path.open() as f: - return yaml.safe_load(f) - - -def setup_logging( - config: t.Mapping[str, t.Any], - default_logging_config: dict[str, t.Any], -) -> None: - """Setup logging. - - Args: - default_logging_config: A default - :py:std:label:`Python logging configuration dictionary`. - config: A plugin configuration dictionary. - """ - logging.config.dictConfig(default_logging_config) - - config = config or {} - metrics_log_level = config.get(METRICS_LOG_LEVEL_SETTING, "INFO").upper() - logging.getLogger(METRICS_LOGGER_NAME).setLevel(metrics_log_level) - - if "SINGER_SDK_LOG_CONFIG" in os.environ: # pragma: no cover - log_config_path = Path(os.environ["SINGER_SDK_LOG_CONFIG"]) - logging.config.dictConfig(load_yaml_logging_config(log_config_path)) diff --git a/singer_sdk/metrics.py b/singer_sdk/metrics.py index 826035b60..3c4b09c6c 100644 --- a/singer_sdk/metrics.py +++ b/singer_sdk/metrics.py @@ -7,14 +7,21 @@ import json import logging import logging.config +import os import typing as t from dataclasses import dataclass, field +from pathlib import Path from time import time +import yaml + +from singer_sdk.helpers._resources import get_package_files + if t.TYPE_CHECKING: from types import TracebackType from singer_sdk.helpers import types + from singer_sdk.helpers._compat import Traversable DEFAULT_LOG_INTERVAL = 60.0 @@ -372,3 +379,48 @@ def sync_timer(stream: str, **tags: t.Any) -> Timer: """ tags[Tag.STREAM] = stream return Timer(Metric.SYNC_DURATION, tags) + + +def _load_yaml_logging_config(path: Traversable | Path) -> t.Any: # noqa: ANN401 + """Load the logging config from the YAML file. + + Args: + path: A path to the YAML file. + + Returns: + The logging config. + """ + with path.open() as f: + return yaml.safe_load(f) + + +def _get_default_config() -> t.Any: # noqa: ANN401 + """Get a logging configuration. + + Returns: + A logging configuration. + """ + filename = "default_logging.yml" + path = get_package_files(__package__) / filename + if path.is_file(): + return _load_yaml_logging_config(path) + + path = get_package_files("singer_sdk").joinpath("default_logging.yml") + return _load_yaml_logging_config(path) + + +def _setup_logging(config: t.Mapping[str, t.Any]) -> None: + """Setup logging. + + Args: + config: A plugin configuration dictionary. + """ + logging.config.dictConfig(_get_default_config()) + + config = config or {} + metrics_log_level = config.get(METRICS_LOG_LEVEL_SETTING, "INFO").upper() + logging.getLogger(METRICS_LOGGER_NAME).setLevel(metrics_log_level) + + if "SINGER_SDK_LOG_CONFIG" in os.environ: + log_config_path = Path(os.environ["SINGER_SDK_LOG_CONFIG"]) + logging.config.dictConfig(_load_yaml_logging_config(log_config_path)) diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 39add257b..1564558cf 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -15,7 +15,6 @@ import click from jsonschema import Draft7Validator -import singer_sdk.logger as singer_logger from singer_sdk import about, metrics from singer_sdk.cli import plugin_cli from singer_sdk.configuration._dict_config import ( @@ -24,7 +23,6 @@ ) from singer_sdk.exceptions import ConfigValidationError from singer_sdk.helpers._classproperty import classproperty -from singer_sdk.helpers._resources import get_package_files from singer_sdk.helpers._secrets import SecretString, is_common_secret_key from singer_sdk.helpers._util import read_json_file from singer_sdk.helpers.capabilities import ( @@ -164,7 +162,7 @@ def __init__( if self._is_secret_config(k): config_dict[k] = SecretString(v) self._config = config_dict - singer_logger.setup_logging(self.config, self.get_default_logging_config()) + metrics._setup_logging(self.config) # noqa: SLF001 self.metrics_logger = metrics.get_metrics_logger() self._validate_config(raise_errors=validate_config) @@ -180,15 +178,6 @@ def setup_mapper(self) -> None: logger=self.logger, ) - def get_default_logging_config(self) -> t.Any: # noqa: ANN401, PLR6301 - """Get a default logging configuration for the plugin. - - Returns: - A logging configuration. - """ - log_config_path = get_package_files("singer_sdk") / "default_logging.yml" - return singer_logger.load_yaml_logging_config(log_config_path) - @property def mapper(self) -> PluginMapper: """Plugin mapper for this tap.