From b7d2fb84d2189c164c2c725a721eaa78b3298696 Mon Sep 17 00:00:00 2001 From: Ben Darwin Date: Thu, 23 Oct 2014 14:02:42 -0400 Subject: [PATCH 1/5] add default values for env variables PYRO_LOGLEVEL, PYRO_SERVERTYPE, PYRO_DETAILED_TRACEBACK --- pydpiper/pipeline.py | 7 +++++++ pydpiper/pipeline_executor.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/pydpiper/pipeline.py b/pydpiper/pipeline.py index 03866fa9..9dca556b 100755 --- a/pydpiper/pipeline.py +++ b/pydpiper/pipeline.py @@ -17,6 +17,13 @@ import threading import Pyro4 +os.environ["PYRO_LOGLEVEL"] = os.environ.get("PYRO_LOGLEVEL", "INFO") + +import Pyro4 + +Pyro4.config.SERVERTYPE = pe.Pyro4.config.SERVERTYPE +Pyro4.config.DETAILED_TRACEBACK = pe.Pyro4.config.DETAILED_TRACEBACK + LOOP_INTERVAL = 5.0 RESPONSE_LATENCY = 100 diff --git a/pydpiper/pipeline_executor.py b/pydpiper/pipeline_executor.py index bff21651..46a7d667 100755 --- a/pydpiper/pipeline_executor.py +++ b/pydpiper/pipeline_executor.py @@ -21,8 +21,13 @@ G_log_file_name = G_prog_name + '-' + G_time_now + '-pid-' + G_proc_id + ".log" os.environ["PYRO_LOGFILE"] = G_log_file_name +os.environ["PYRO_LOGLEVEL"] = os.environ.get("PYRO_LOGLEVEL", "INFO") + import Pyro4 +Pyro4.config.SERVERTYPE = "multiplex" +Pyro4.config.DETAILED_TRACEBACK = os.environ.get("PYRO_DETAILED_TRACEBACK", True) + WAIT_TIMEOUT = 5.0 HEARTBEAT_INTERVAL = 30.0 From 60f5a851cb821f2676d9a0fafffb7a2dd0da42da Mon Sep 17 00:00:00 2001 From: Ben Darwin Date: Thu, 23 Oct 2014 14:19:43 -0400 Subject: [PATCH 2/5] change os.environ.get to os.getenv --- pydpiper/pipeline.py | 2 +- pydpiper/pipeline_executor.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pydpiper/pipeline.py b/pydpiper/pipeline.py index 9dca556b..b455006c 100755 --- a/pydpiper/pipeline.py +++ b/pydpiper/pipeline.py @@ -17,7 +17,7 @@ import threading import Pyro4 -os.environ["PYRO_LOGLEVEL"] = os.environ.get("PYRO_LOGLEVEL", "INFO") +os.environ["PYRO_LOGLEVEL"] = os.getenv("PYRO_LOGLEVEL", "INFO") import Pyro4 diff --git a/pydpiper/pipeline_executor.py b/pydpiper/pipeline_executor.py index 46a7d667..cefb9f0e 100755 --- a/pydpiper/pipeline_executor.py +++ b/pydpiper/pipeline_executor.py @@ -21,12 +21,12 @@ G_log_file_name = G_prog_name + '-' + G_time_now + '-pid-' + G_proc_id + ".log" os.environ["PYRO_LOGFILE"] = G_log_file_name -os.environ["PYRO_LOGLEVEL"] = os.environ.get("PYRO_LOGLEVEL", "INFO") +os.environ["PYRO_LOGLEVEL"] = os.getenv("PYRO_LOGLEVEL", "INFO") import Pyro4 Pyro4.config.SERVERTYPE = "multiplex" -Pyro4.config.DETAILED_TRACEBACK = os.environ.get("PYRO_DETAILED_TRACEBACK", True) +Pyro4.config.DETAILED_TRACEBACK = os.getenv("PYRO_DETAILED_TRACEBACK", True) WAIT_TIMEOUT = 5.0 HEARTBEAT_INTERVAL = 30.0 From 2f43abc55b98639113c084aac1aa22cd0f41a603 Mon Sep 17 00:00:00 2001 From: Ben Darwin Date: Fri, 24 Oct 2014 11:57:06 -0400 Subject: [PATCH 3/5] remove duplicate Pyro import --- pydpiper/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydpiper/pipeline.py b/pydpiper/pipeline.py index b455006c..ef30dc59 100755 --- a/pydpiper/pipeline.py +++ b/pydpiper/pipeline.py @@ -15,7 +15,6 @@ import pipeline_executor as pe import logging import threading -import Pyro4 os.environ["PYRO_LOGLEVEL"] = os.getenv("PYRO_LOGLEVEL", "INFO") From 0cbd09b4d82d7cfa0e13326c347c21361ea3ae16 Mon Sep 17 00:00:00 2001 From: Ben Darwin Date: Fri, 24 Oct 2014 15:28:30 -0400 Subject: [PATCH 4/5] update documentation --- INSTALL | 6 ------ README | 4 ++++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/INSTALL b/INSTALL index 2a2b2656..58b2f668 100644 --- a/INSTALL +++ b/INSTALL @@ -15,12 +15,6 @@ graphviz - http://www.graphviz.org/ pygraphviz - http://pygraphviz.github.io/ minc-stuffs - https://github.com/Mouse-Imaging-Centre/minc-stuffs -The following environment variables should be set when running pydpiper code: - -PYRO_DETAILED_TRACEBACK=True -PYRO_SERVERTYPE=multiplex -PYRO_LOGLEVEL=INFO - In Ubuntu (tested on 12.04), run: apt-get install python-networkx pip install pyro diff --git a/README b/README index fbdd1fc7..1546fbd3 100644 --- a/README +++ b/README @@ -9,5 +9,9 @@ The code for executing the pipeline can be found in pipeline_executor.py. sge_batch is a perl script (written independently of this code) that is required to use the --queue=sge option currently written into pydpiper. It may be rewritten or moved to a separate subdirectory in the future. For now, this script should reside in a directory in the user's path. +You can use environment variables to override our configuration defaults for the underlying Pyro library, except for +$PYRO_SERVERTYPE and $PYRO_LOGFILE; in particular, you may wish to change $PYRO_LOGLEVEL, since this also controls +the verbosity of some of the application's own logging. See the Pyro4 documentation for more options. + Application modules that utilize the pipeline class definitions are currently in applications folder. These applications may be moved to a separate repository at a later date. From 7bd4c21077d65a41eb232ad8b9cef98cd4a067e0 Mon Sep 17 00:00:00 2001 From: Matthijs van Eede Date: Fri, 24 Oct 2014 16:55:30 -0400 Subject: [PATCH 5/5] updated version and NEWS --- NEWS | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index ed0a63f5..9387a25e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +New in Version 1.10 +=================== +* PYRO environment variables are handled inside the code. The SERVERTYPE is + set statically to multiplex (there are potential issues with the thread option) + and the LOGLEVEL and DETAILED_TRACEBACK can be overridden by environment + variables (see the README) + New in Version 1.9 ================== * MAGeT uses the originally optimized defaults again. Hardcoded paths to the diff --git a/setup.py b/setup.py index 59b5778a..02461ff1 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup setup(name='pydpiper', - version='1.9', + version='1.10', license='Modified BSD', description='Python code for flexible pipeline control', long_description = 'Python code for flexible pipeline control',