From 81677d4d85e143ab5b4f22bda32ce736c7c72d9a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 20 Nov 2024 09:00:35 -0500 Subject: [PATCH 01/10] enh: make logging level of the django app configurable ATM we do not have a single DEBUG level log statement, but I hope that logging would be enriched. Adding such control would also allow to make log less talkative if so desired by setting it to WARNING level. But also would simplify addition and use of new log statements --- dandiapi/api/apps.py | 7 ++++++- dandiapi/settings.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 93d3f29d6..0e89fa673 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,13 +50,18 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 + logging_level = logging.INFO + if hasattr(settings, 'DANDI_LOG_LEVEL'): + logging_level = getattr(logging, settings.DANDI_LOG_LEVEL.upper()) + logging.getLogger(__name__.split('.')[0]).setLevel(logging_level) + if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( dsn=settings.SENTRY_DSN, environment=settings.SENTRY_ENVIRONMENT, release=settings.SENTRY_RELEASE, integrations=[ - LoggingIntegration(level=logging.INFO, event_level=logging.WARNING), + LoggingIntegration(level=logging_level, event_level=logging.WARNING), DjangoIntegration(), CeleryIntegration(), ], diff --git a/dandiapi/settings.py b/dandiapi/settings.py index 0e87dfc2d..7ae1db6e2 100644 --- a/dandiapi/settings.py +++ b/dandiapi/settings.py @@ -89,6 +89,7 @@ def mutate_configuration(configuration: type[ComposedConfiguration]): DANDI_DANDISETS_BUCKET_PREFIX = values.Value(default='', environ=True) DANDI_DANDISETS_LOG_BUCKET_NAME = values.Value(environ_required=True) DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = values.Value(environ_required=True) + DANDI_LOG_LEVEL = values.Value(environ_required=True) DANDI_ZARR_PREFIX_NAME = values.Value(default='zarr', environ=True) # Mainly applies to unembargo From d9f5626c87e3ea46fbfce54d7c644c147da739d6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 3 Dec 2024 10:13:48 -0500 Subject: [PATCH 02/10] Do pass DJANGO_DANDI_LOG_LEVEL in tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 26e310864..d15e58095 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,7 @@ passenv = DJANGO_DANDI_API_URL DJANGO_DANDI_JUPYTERHUB_URL DJANGO_DANDI_DEV_EMAIL + DJANGO_DANDI_LOG_LEVEL DANDI_ALLOW_LOCALHOST_URLS extras = dev @@ -69,6 +70,7 @@ passenv = DJANGO_DANDI_API_URL DJANGO_DANDI_JUPYTERHUB_URL DJANGO_DANDI_DEV_EMAIL + DJANGO_DANDI_LOG_LEVEL extras = dev commands = From 01cd48ac4c4e3e75779e8f878ba514f5ffb63245 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 3 Dec 2024 10:15:32 -0500 Subject: [PATCH 03/10] Make DANDI_LOG_LEVEL not required to be passed via environ - default to INFO --- dandiapi/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandiapi/settings.py b/dandiapi/settings.py index 7ae1db6e2..500e0a533 100644 --- a/dandiapi/settings.py +++ b/dandiapi/settings.py @@ -89,7 +89,7 @@ def mutate_configuration(configuration: type[ComposedConfiguration]): DANDI_DANDISETS_BUCKET_PREFIX = values.Value(default='', environ=True) DANDI_DANDISETS_LOG_BUCKET_NAME = values.Value(environ_required=True) DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME = values.Value(environ_required=True) - DANDI_LOG_LEVEL = values.Value(environ_required=True) + DANDI_LOG_LEVEL = values.Value(default='INFO', environ=True) DANDI_ZARR_PREFIX_NAME = values.Value(default='zarr', environ=True) # Mainly applies to unembargo From 14ba71bd5eb68001da65289cf2d70efa08326827 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 13 Dec 2024 13:34:59 -0500 Subject: [PATCH 04/10] Eliminate default value and conditional use These are not needed here, as they are already part of the setting definition (that is, `settings.DANDI_LOG_LEVEL` always exists and has the correct default value if it is not set explicitly). --- dandiapi/api/apps.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 0e89fa673..97819e3f2 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,10 +50,8 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - logging_level = logging.INFO - if hasattr(settings, 'DANDI_LOG_LEVEL'): - logging_level = getattr(logging, settings.DANDI_LOG_LEVEL.upper()) - logging.getLogger(__name__.split('.')[0]).setLevel(logging_level) + logging_level = getattr(logging, settings.DANDI_LOG_LEVEL.upper()) + logging.getLogger(__name__.split('.')[0]).setLevel(logging_level) if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( From e1b66da87ba76a7ef8b120996d88c39f31b184e1 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 13 Dec 2024 13:43:00 -0500 Subject: [PATCH 05/10] Pass the string value directly to `setLevel()` The [`logging` module docs](https://docs.python.org/3/library/logging.html#logging.Logger.setLevel) explain that `setLevel()` can handle the string representations of the logging levels. This allows the setting value to be directly tied to what is passed in, and `setLevel()` also does a "validation" on the value and will refuse to process levels that don't exist. --- dandiapi/api/apps.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 97819e3f2..4136e257a 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,8 +50,7 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - logging_level = getattr(logging, settings.DANDI_LOG_LEVEL.upper()) - logging.getLogger(__name__.split('.')[0]).setLevel(logging_level) + logging.getLogger(__name__.split('.')[0]).setLevel(settings.DANDI_LOG_LEVEL) if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( @@ -59,7 +58,7 @@ def ready(self): environment=settings.SENTRY_ENVIRONMENT, release=settings.SENTRY_RELEASE, integrations=[ - LoggingIntegration(level=logging_level, event_level=logging.WARNING), + LoggingIntegration(level=settings.DANDI_LOG_LEVEL, event_level=logging.WARNING), DjangoIntegration(), CeleryIntegration(), ], From 6addfacce0d6fac0bf11c0da77e038be412ce344 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 13 Dec 2024 13:45:13 -0500 Subject: [PATCH 06/10] Use standard practice for retrieving module level logger --- dandiapi/api/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 4136e257a..4782650f7 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,7 +50,7 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - logging.getLogger(__name__.split('.')[0]).setLevel(settings.DANDI_LOG_LEVEL) + logging.getLogger(__name__).setLevel(settings.DANDI_LOG_LEVEL) if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( From 74697eaf862d122153a1f29357c400cfb58ade1f Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 13 Dec 2024 14:41:57 -0500 Subject: [PATCH 07/10] Maintain original Sentry configuration This needs to be `logging.INFO` so that Sentry can propery construct breadcrumbs within its error reports. --- dandiapi/api/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 4782650f7..be760ff62 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -58,7 +58,7 @@ def ready(self): environment=settings.SENTRY_ENVIRONMENT, release=settings.SENTRY_RELEASE, integrations=[ - LoggingIntegration(level=settings.DANDI_LOG_LEVEL, event_level=logging.WARNING), + LoggingIntegration(level=logging.INFO, event_level=logging.WARNING), DjangoIntegration(), CeleryIntegration(), ], From b07bfe126db504ab7c879b37eefbad647c2baac3 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Fri, 13 Dec 2024 14:42:42 -0500 Subject: [PATCH 08/10] Set global logging level on the root logger --- dandiapi/api/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index be760ff62..a0774d86c 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,7 +50,7 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - logging.getLogger(__name__).setLevel(settings.DANDI_LOG_LEVEL) + logging.getLogger().setLevel(settings.DANDI_LOG_LEVEL) if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( From dc3549dd1881a963b72e9bd03395ecef64ab9a74 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Thu, 19 Dec 2024 11:28:20 -0500 Subject: [PATCH 09/10] Restrict default log level to just DANDI loggers --- dandiapi/api/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index a0774d86c..2b17f67f6 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,7 +50,8 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - logging.getLogger().setLevel(settings.DANDI_LOG_LEVEL) + # Configure the logging level on just DANDI loggers. + logging.getLogger(__name__.split('.')[0]).setLevel(settings.DANDI_LOG_LEVEL) if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( From 78c8827d658037681e753ad38b00bd7eaa37dd21 Mon Sep 17 00:00:00 2001 From: Roni Choudhury Date: Thu, 2 Jan 2025 15:23:04 -0500 Subject: [PATCH 10/10] Move logger level setting to dandiapi top level module --- dandiapi/__init__.py | 6 ++++++ dandiapi/api/apps.py | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dandiapi/__init__.py b/dandiapi/__init__.py index aabca329d..dc3bce32b 100644 --- a/dandiapi/__init__.py +++ b/dandiapi/__init__.py @@ -1,6 +1,9 @@ from __future__ import annotations from importlib.metadata import version +import logging + +from django.conf import settings # This project module is imported for us when Django starts. To ensure that Celery app is always # defined prior to any shared_task definitions (so those tasks will bind to the app), import @@ -8,3 +11,6 @@ from .celery import app as _celery_app # noqa: F401 __version__ = version('dandiapi') + +# Configure the logging level on all DANDI loggers. +logging.getLogger(__name__).setLevel(settings.DANDI_LOG_LEVEL) diff --git a/dandiapi/api/apps.py b/dandiapi/api/apps.py index 2b17f67f6..93d3f29d6 100644 --- a/dandiapi/api/apps.py +++ b/dandiapi/api/apps.py @@ -50,9 +50,6 @@ def ready(self): import dandiapi.api.checks # noqa: F401, RUF100 import dandiapi.api.signals # noqa: F401 - # Configure the logging level on just DANDI loggers. - logging.getLogger(__name__.split('.')[0]).setLevel(settings.DANDI_LOG_LEVEL) - if hasattr(settings, 'SENTRY_DSN'): sentry_sdk.init( dsn=settings.SENTRY_DSN,