test: Update test_settings.py

noxfile.py

@@ -34,7 +34,7 @@ def mypy(session):
 @nox.session(python=PYTHON_VERSIONS, tags=["test"])
 def pytest(session):
     session.install(".[all]")
-    session.install("pytest")
+    session.install(".[test]")
     session.run("pytest")
 
 

setup.py

@@ -70,16 +70,22 @@
         ],
         "dev": [
             "cluster_utils[lint]",
+            "cluster_utils[test]",
             "absolufy-imports",
             "nox>=2022.8.7",
             "pre-commit",
-            "pytest",
         ],
         "mypy": [
             "mypy",
             "pandas-stubs",
+            "tomli-w",
             "types-colorama",
             "types-tqdm",
+            "types-PyYaml",
+        ],
+        "test": [
+            "pytest",
+            "tomli-w",
         ],
         "docs": [
             "sphinx",

tests/test_settings.py

@@ -1,7 +1,10 @@
+import argparse
 import json
 import pathlib
 
 import pytest
+import tomli_w  # FIXME add to test dependencies
+import yaml
 
 from cluster_utils import constants
 from cluster_utils import settings as s
@@ -28,7 +31,7 @@ def base_config() -> dict:
     return {
         "optimization_procedure_name": "test_grid_search",
         "run_in_working_dir": True,
-        "git_params": {"branch": "master", "commit": None},
+        "git_params": {"branch": "master"},
         "generate_report": "never",
         "script_relative_path": "examples/basic/main_no_fail.py",
         "remove_jobs_dir": True,
@@ -39,7 +42,6 @@ def base_config() -> dict:
         "cluster_requirements": {
             "request_cpus": 1,
             "request_gpus": 0,
-            "cuda_requirement": None,
             "memory_in_mb": 16000,
             "bid": 800,
         },
@@ -52,45 +54,178 @@ def base_config() -> dict:
             "fn_args.sharp_penalty": False,
         },
         "restarts": 1,
-        "samples": None,
+        "samples": 20,
         "hyperparam_list": [
             {"param": "fn_args.u", "values": [-0.5, 0.5]},
             {"param": "fn_args.v", "values": [10, 50]},
         ],
     }
 
 
-def test_read_params_from_cmdline__basic(tmp_path, base_config):
+# FIXME needed testcases:
+# - job script errors
+
+
+def test_init_main_script_argument_parser():
+    parser = s.init_main_script_argument_parser(description="foo")
+
+    # basic (only file)
+    args = parser.parse_args(["file.toml"])
+    assert hasattr(args, "settings_file")
+    assert hasattr(args, "settings")
+    assert args.settings_file == pathlib.Path("file.toml")
+    assert args.settings == []
+
+    # file + cmd line args
+    args = parser.parse_args(["file.toml", "foo=42", "bar='hi'"])
+    assert hasattr(args, "settings_file")
+    assert hasattr(args, "settings")
+    assert args.settings_file == pathlib.Path("file.toml")
+    assert args.settings == ["foo=42", "bar='hi'"]
+
+
+@pytest.mark.parametrize(
+    "config_file_format",
+    [("json", "w", json.dump), ("yml", "w", yaml.dump), ("toml", "wb", tomli_w.dump)],
+)
+def test_read_main_script_params_from_args__basic(
+    tmp_path, base_config, config_file_format
+):
+    config_file_ext, config_file_write_mode, config_file_dump = config_file_format
+
     # save config to file
-    config_file = tmp_path / "config.json"
-    with open(config_file, "w") as f:
-        json.dump(base_config, f)
+    config_file = tmp_path / f"config.{config_file_ext}"
+    with open(config_file, config_file_write_mode) as f:
+        config_file_dump(base_config, f)
 
-    cmd_line = ["test", str(config_file)]
-    params = s.read_params_from_cmdline(cmd_line)
+    args = argparse.Namespace(settings_file=config_file, settings=[])
+    params = s.read_main_script_params_from_args(args)
 
     # just a basic check to see if the file was read
     assert params["optimization_procedure_name"] == "test_grid_search"
+    assert params["fixed_params"]["test_resume"] is False
+    assert params["generate_report"] is s.GenerateReportSetting.NEVER
+    # TODO more checks to verify the other hooks are doing their jobs
 
 
-def test_read_params_from_cmdline__with_hooks(tmp_path, base_config):
+def test_read_main_script_params_from_args__with_cmdline_overwrites(
+    tmp_path, base_config
+):
     # save config to file
     config_file = tmp_path / "config.json"
     with open(config_file, "w") as f:
         json.dump(base_config, f)
 
-    cmd_line = ["test", str(config_file)]
-    params = s.read_params_from_cmdline(
-        cmd_line,
-        pre_unpack_hooks=_PRE_UNPACK_HOOKS,
-        post_unpack_hooks=_POST_UNPACK_HOOKS,
-        verbose=False,  # TODO fails if verbose is true
+    args = argparse.Namespace(
+        settings_file=config_file,
+        settings=["optimization_procedure_name='foo'", "fixed_params.test_resume=True"],
     )
+    params = s.read_main_script_params_from_args(args)
 
     # just a basic check to see if the file was read
-    assert params["optimization_procedure_name"] == "test_grid_search"
+    assert params["optimization_procedure_name"] == "foo"
+    assert params["fixed_params"]["test_resume"] is True
     assert params["generate_report"] is s.GenerateReportSetting.NEVER
-    # TODO more checks to verify the other hooks are doing their jobs
+
+
+def test_read_params_from_cmdline__dict():
+    argv = ["test", "--parameter-dict={'foo': 'bar', 'one': {'two': 13}}"]
+    params = s.read_params_from_cmdline(cmd_line=argv)
+
+    assert "foo" in params
+    assert "one" in params
+    assert "two" in params["one"]
+    assert params["foo"] == "bar"
+    assert params["one"]["two"] == 13
+
+
+def test_read_params_from_cmdline__dict_and_overwrite():
+    argv = [
+        "test",
+        "--parameter-dict",
+        "{'foo': 'bar', 'one': {'two': 13}}",
+        "--parameters",
+        "foo='blub'",
+        "three=3",
+    ]
+    params = s.read_params_from_cmdline(cmd_line=argv)
+
+    assert "foo" in params
+    assert params["foo"] == "blub"
+    assert "one" in params
+    assert "two" in params["one"]
+    assert params["one"]["two"] == 13
+    assert "three" in params
+    assert params["three"] == 3
+
+
+def test_read_params_from_cmdline__file(tmp_path):
+    # save config to file
+    config_file = tmp_path / "config.json"
+    with open(config_file, "w") as f:
+        json.dump({"foo": "bar", "one": {"two": 13}}, f)
+
+    argv = ["test", f"--parameter-file={config_file}"]
+    params = s.read_params_from_cmdline(cmd_line=argv)
+
+    assert "foo" in params
+    assert "one" in params
+    assert "two" in params["one"]
+    assert params["foo"] == "bar"
+    assert params["one"]["two"] == 13
+
+
+def test_read_params_from_cmdline__file_and_overwrite(tmp_path):
+    # save config to file
+    config_file = tmp_path / "config.json"
+    with open(config_file, "w") as f:
+        json.dump({"foo": "bar", "one": {"two": 13}}, f)
+
+    argv = [
+        "test",
+        "--parameter-file",
+        str(config_file),
+        "--parameters",
+        "foo='blub'",
+        "three=3",
+    ]
+    params = s.read_params_from_cmdline(cmd_line=argv)
+
+    assert "foo" in params
+    assert params["foo"] == "blub"
+    assert "one" in params
+    assert "two" in params["one"]
+    assert params["one"]["two"] == 13
+    assert "three" in params
+    assert params["three"] == 3
+
+
+def test_read_params_from_cmdline__errors():
+    argv = [
+        "test",
+        "--server-connection-info='notadictionary'",
+        "--parameter-dict={'foo': 13}",
+    ]
+    with pytest.raises(
+        ValueError, match="'--server-connection-info' must be a dictionary."
+    ):
+        s.read_params_from_cmdline(cmd_line=argv)
+
+    argv = [
+        "test",
+        "--server-connection-info={'wrong': 1, 'keys': 2}",
+        "--parameter-dict={'foo': 13}",
+    ]
+    with pytest.raises(
+        ValueError,
+        # order of keys in the error message is not explicitly defined, so just match .*
+        match=r"'--server-connection-info' must contain keys \{.*\}",
+    ):
+        s.read_params_from_cmdline(cmd_line=argv)
+
+    argv = ["test", "--parameter-dict='notadictionary'"]
+    with pytest.raises(ValueError, match="'--parameter-dict' must be a dictionary."):
+        s.read_params_from_cmdline(cmd_line=argv)
 
 
 def test_read_params_from_cmdline__working_dir(tmp_path, base_config):
@@ -104,17 +239,14 @@ def test_read_params_from_cmdline__working_dir(tmp_path, base_config):
     with open(config_file, "w") as f:
         json.dump(base_config, f)
 
-    cmd_line = ["test", str(config_file)]
+    cmd_line = ["test", "--parameter-file", str(config_file)]
     params = s.read_params_from_cmdline(
         cmd_line,
-        pre_unpack_hooks=_PRE_UNPACK_HOOKS,
-        post_unpack_hooks=_POST_UNPACK_HOOKS,
         verbose=False,
     )
 
     # just a basic check to see if the file was read
     assert params["optimization_procedure_name"] == "test_grid_search"
-    assert params["generate_report"] is s.GenerateReportSetting.NEVER
 
     # verify that settings file was written
     output_settings_file = working_dir / constants.JSON_SETTINGS_FILE