Add support for overriding st2.conf settings via env vars (#6277)

CHANGELOG.rst

@@ -86,6 +86,13 @@ Added
   following to set system user to the current user: `export ST2TESTS_SYSTEM_USER=$(id -un)` #6242
   Contributed by @cognifloyd
 
+* Added experimental support for setting conf vars via environment variables. All settings in `st2.conf` can be
+  overriden via enviornment vars in the format: `ST2_<conf section>__<option name>` (env vars are upper cased)
+  For example, the `[database].password` setting in `st2.conf` could be overriden using `ST2_DATABASE__PASSWORD`.
+  This new feature is based on oslo_config's environment support, but patches it to use the `ST2_` prefix.
+  If you experience any issues when using this experimental feature, please file an issue. #6277
+  Contributed by @cognifloyd
+
 3.8.1 - December 13, 2023
 -------------------------
 Fixed

st2actions/st2actions/config.py

@@ -28,6 +28,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(CONF)
     CONF(
         args=args,
         version=VERSION_STRING,

st2actions/st2actions/notifier/config.py

@@ -21,10 +21,9 @@
 from st2common.constants.system import VERSION_STRING
 from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH
 
-CONF = cfg.CONF
-
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2actions/st2actions/scheduler/config.py

@@ -27,6 +27,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=sys_constants.VERSION_STRING,

st2actions/st2actions/workflows/config.py

@@ -23,6 +23,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=sys_constants.VERSION_STRING,

st2api/st2api/config.py

@@ -32,6 +32,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2api/tests/integration/test_gunicorn_configs.py

@@ -22,6 +22,7 @@
 import eventlet
 from eventlet.green import subprocess
 
+import st2tests.config
 from st2common.models.utils import profiling
 from st2common.util.shell import kill_process
 from st2tests.base import IntegrationTestCase
@@ -41,6 +42,9 @@ def test_st2api_wsgi_entry_point(self):
         )
         env = os.environ.copy()
         env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
         process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid)
         try:
             self.add_process(process=process)
@@ -60,6 +64,9 @@ def test_st2auth(self):
         )
         env = os.environ.copy()
         env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
         process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid)
         try:
             self.add_process(process=process)

st2auth/st2auth/config.py

@@ -28,6 +28,7 @@
 
 
 def parse_args(args=None):
+    st2cfg.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2common/st2common/config.py

@@ -19,6 +19,7 @@
 import sys
 
 from oslo_config import cfg
+from oslo_config.sources._environment import EnvironmentConfigurationSource
 
 from st2common.constants.system import VERSION_STRING
 from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH
@@ -900,7 +901,20 @@ def register_opts(ignore_errors=False):
     )
 
 
+class St2EnvironmentConfigurationSource(EnvironmentConfigurationSource):
+    @staticmethod
+    def get_name(group_name, option_name):
+        group_name = group_name or "DEFAULT"
+        return "ST2_{}__{}".format(group_name.upper(), option_name.upper())
+
+
+def use_st2_env_vars(conf: cfg.ConfigOpts) -> None:
+    # Override oslo_config's 'OS_' env var prefix with 'ST2_'.
+    conf._env_driver = St2EnvironmentConfigurationSource()
+
+
 def parse_args(args=None, ignore_errors=False):
+    use_st2_env_vars(cfg.CONF)
     register_opts(ignore_errors=ignore_errors)
     cfg.CONF(
         args=args,

st2common/tests/integration/test_register_content_script.py

@@ -19,6 +19,7 @@
 import sys
 import glob
 
+import st2tests.config
 from st2tests.base import IntegrationTestCase
 from st2common.util.shell import run_command
 from st2tests import config as test_config
@@ -56,7 +57,7 @@ def test_register_from_pack_success(self):
             "--register-runner-dir=%s" % (runner_dirs),
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 3 actions.", stderr)
         self.assertEqual(exit_code, 0)
 
@@ -71,7 +72,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
             "--register-no-fail-on-failure",
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, _ = run_command(cmd=cmd)
+        exit_code, _, _ = self._run_command(cmd=cmd)
         self.assertEqual(exit_code, 0)
 
         # Fail on failure, should fail
@@ -81,7 +82,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
             "--register-fail-on-failure",
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn('Directory "doesntexistblah" doesn\'t exist', stderr)
         self.assertEqual(exit_code, 1)
 
@@ -97,7 +98,7 @@ def test_register_from_pack_action_metadata_fails_validation(self):
         ]
 
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 actions.", stderr)
         self.assertEqual(exit_code, 0)
 
@@ -109,7 +110,7 @@ def test_register_from_pack_action_metadata_fails_validation(self):
             "--register-runner-dir=%s" % (runner_dirs),
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("object has no attribute 'get'", stderr)
         self.assertEqual(exit_code, 1)
 
@@ -127,7 +128,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self):
             "-v",
             "--register-sensors",
         ]
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 sensors.", stderr, "Actual stderr: %s" % (stderr))
         self.assertEqual(exit_code, 0)
 
@@ -139,7 +140,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self):
             "--register-all",
             "--register-no-fail-on-failure",
         ]
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 actions.", stderr)
         self.assertIn("Registered 0 sensors.", stderr)
         self.assertIn("Registered 0 rules.", stderr)
@@ -155,7 +156,7 @@ def test_register_all_and_register_setup_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registering actions", stderr, "Actual stderr: %s" % (stderr))
         self.assertIn("Registering rules", stderr)
         self.assertIn("Setup virtualenv for %s pack(s)" % ("1"), stderr)
@@ -170,7 +171,7 @@ def test_register_setup_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
@@ -186,7 +187,7 @@ def test_register_recreate_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
@@ -200,9 +201,17 @@ def test_register_recreate_virtualenvs(self):
             "--register-recreate-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Virtualenv successfully removed.", stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
         self.assertEqual(exit_code, 0)
+
+    @staticmethod
+    def _run_command(cmd):
+        env = os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
+        return run_command(cmd=cmd, env=env)

st2common/tests/integration/test_service_setup_log_level_filtering.py

@@ -21,6 +21,7 @@
 import eventlet
 from eventlet.green import subprocess
 
+import st2tests.config
 from st2tests.base import IntegrationTestCase
 from st2tests.fixtures.conf.fixture import FIXTURE_PATH as CONF_FIXTURES_PATH
 
@@ -223,13 +224,18 @@ def test_kombu_heartbeat_tick_log_messages_are_excluded(self):
         stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
         self.assertNotIn("heartbeat_tick", stdout)
 
-    def _start_process(self, config_path, env=None):
+    @staticmethod
+    def _start_process(config_path, env=None):
         cmd = CMD + [config_path]
         cwd = os.path.abspath(os.path.join(BASE_DIR, "../../../"))
         cwd = os.path.abspath(cwd)
+        env = env or os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
         process = subprocess.Popen(
             cmd,
-            env=env or os.environ.copy(),
+            env=env,
             cwd=cwd,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,

st2reactor/st2reactor/cmd/trigger_re_fire.py

@@ -48,6 +48,7 @@ def _parse_config():
     CONF.register_cli_opts(cli_opts)
     st2cfg.register_opts(ignore_errors=False)
 
+    st2cfg.use_st2_env_vars(CONF)
     CONF(args=sys.argv[1:])
 
 

st2reactor/st2reactor/garbage_collector/config.py

@@ -27,6 +27,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2reactor/st2reactor/rules/config.py

@@ -25,6 +25,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2reactor/st2reactor/sensor/config.py

@@ -26,6 +26,7 @@
 
 
 def parse_args(args=None):
+    st2cfg.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2reactor/st2reactor/timer/config.py

@@ -25,6 +25,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2reactor/tests/integration/test_garbage_collector.py

@@ -20,6 +20,7 @@
 import signal
 import datetime
 
+import st2tests.config
 from st2common.util import concurrency
 from st2common.constants import action as action_constants
 from st2common.util import date as date_utils
@@ -274,12 +275,17 @@ def _create_inquiry(self, ttl, timestamp):
 
     def _start_garbage_collector(self):
         subprocess = concurrency.get_subprocess_module()
+        env = os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
         process = subprocess.Popen(
             CMD_INQUIRY,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             shell=False,
             preexec_fn=os.setsid,
+            env=env,
         )
         self.add_process(process=process)
         return process

st2reactor/tests/integration/test_sensor_container.py

@@ -244,13 +244,18 @@ def test_single_sensor_mode(self):
 
     def _start_sensor_container(self, cmd=DEFAULT_CMD):
         subprocess = concurrency.get_subprocess_module()
+        env = os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
+        env.update(st2tests.config.mq_opts_as_env_vars())
+        env.update(st2tests.config.coord_opts_as_env_vars())
         print("Using command: %s" % (" ".join(cmd)))
         process = subprocess.Popen(
             cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             shell=False,
             preexec_fn=os.setsid,
+            env=env,
         )
         self.add_process(process=process)
         return process

st2stream/st2stream/config.py

@@ -32,6 +32,7 @@
 
 
 def parse_args(args=None):
+    common_config.use_st2_env_vars(cfg.CONF)
     cfg.CONF(
         args=args,
         version=VERSION_STRING,

st2tests/st2tests/base.py

@@ -565,6 +565,11 @@ class IntegrationTestCase(TestCase):
 
     processes = {}
 
+    @classmethod
+    def setUpClass(cls):
+        # this prepares the vars for use in configuring the subprocesses via env var
+        tests_config.parse_args()
+
     def setUp(self):
         super(IntegrationTestCase, self).setUp()
         self._stop_running_processes()