feat: log from file conf

Signed-off-by: Avik Basu <ab93@users.noreply.github.com>

log.conf

@@ -0,0 +1,27 @@
+[loggers]
+keys=root, pllogger
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=consoleFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[logger_pllogger]
+level=ERROR
+handlers=consoleHandler
+qualname=pytorch_lightning
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=consoleFormatter
+
+[formatter_consoleFormatter]
+format=%(asctime)s - %(thread)d - %(levelname)s - %(message)s
+class=logging.Formatter

numalogic/udfs/__init__.py

@@ -1,13 +1,24 @@
 import logging
 import os
 
+from numalogic._constants import BASE_DIR
 from numalogic.udfs._base import NumalogicUDF
+from numalogic.udfs._config import StreamConf, PipelineConf, load_pipeline_conf
+from numalogic.udfs.factory import UDFFactory, ServerFactory
 from numalogic.udfs.inference import InferenceUDF
-from numalogic.udfs.trainer import TrainerUDF
-from numalogic.udfs.preprocess import PreprocessUDF
 from numalogic.udfs.postprocess import PostprocessUDF
-from numalogic.udfs.factory import UDFFactory, ServerFactory
-from numalogic.udfs._config import StreamConf, PipelineConf, load_pipeline_conf
+from numalogic.udfs.preprocess import PreprocessUDF
+from numalogic.udfs.trainer import TrainerUDF
+
+
+def set_logger() -> None:
+    """Sets the logger for the UDFs."""
+    logging.config.fileConfig(
+        fname=os.path.join(BASE_DIR, "log.conf"),
+        disable_existing_loggers=False,
+    )
+    if os.getenv("DEBUG", "false").lower() == "true":
+        logging.getLogger("root").setLevel(logging.DEBUG)
 
 
 __all__ = [
@@ -21,10 +32,5 @@
     "PipelineConf",
     "load_pipeline_conf",
     "ServerFactory",
+    "set_logger",
 ]
-
-
-if os.getenv("DEBUG"):
-    logging.basicConfig(level=logging.DEBUG)
-else:
-    logging.basicConfig(level=logging.INFO)

numalogic/udfs/__main__.py

@@ -4,7 +4,7 @@
 
 from numalogic._constants import BASE_CONF_DIR
 from numalogic.connectors.redis import get_redis_client_from_conf
-from numalogic.udfs import load_pipeline_conf, UDFFactory, ServerFactory
+from numalogic.udfs import load_pipeline_conf, UDFFactory, ServerFactory, set_logger
 
 LOGGER = logging.getLogger(__name__)
 CONF_FILE_PATH = os.getenv(
@@ -13,6 +13,7 @@
 
 
 if __name__ == "__main__":
+    set_logger()
     step = sys.argv[1]
 
     try:

tests/udfs/test_trainer.py

@@ -18,14 +18,13 @@
 from numalogic.udfs import StreamConf, PipelineConf
 from numalogic.udfs.trainer import TrainerUDF
 
-
-logging.basicConfig(level=logging.DEBUG)
-
-
 REDIS_CLIENT = FakeStrictRedis(server=FakeServer())
 KEYS = ["service-mesh", "1", "2"]
 
 
+logging.basicConfig(level=logging.DEBUG)
+
+
 def mock_druid_fetch_data(nrows=5000):
     """Mock druid fetch data."""
     return pd.read_csv(