diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py index 467aa5294..6fa5a864c 100644 --- a/looper/cli_pydantic.py +++ b/looper/cli_pydantic.py @@ -227,6 +227,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): exclusion_flag=subcommand_args.exc_flag, ) as prj: if subcommand_name in ["run", "rerun"]: + rerun = subcommand_name == "rerun" run = Runner(prj) try: # compute_kwargs = _proc_resources_spec(args) @@ -234,7 +235,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): # TODO Shouldn't top level args and subcommand args be accessible on the same object? return run( - subcommand_args, top_level_args=args, rerun=False, **compute_kwargs + subcommand_args, top_level_args=args, rerun=rerun, **compute_kwargs ) except SampleFailedException: sys.exit(1) diff --git a/looper/conductor.py b/looper/conductor.py index bf4f78580..52e921173 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -309,6 +309,7 @@ def add_sample(self, sample, rerun=False): if sample_statuses: status_str = ", ".join(sample_statuses) failed_flag = any("failed" in x for x in sample_statuses) + waiting_flag = any("waiting" in x for x in sample_statuses) if self.ignore_flags: msg = f"> Found existing status: {status_str}. Ignoring." else: # this pipeline already has a status @@ -318,11 +319,11 @@ def add_sample(self, sample, rerun=False): use_this_sample = False if rerun: # Rescue the sample if rerun requested, and failed flag is found - if failed_flag: + if failed_flag or waiting_flag: msg = f"> Re-running failed sample. Status: {status_str}" use_this_sample = True else: - msg = f"> Skipping sample because rerun requested, but no failed flag found. Status: {status_str}" + msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found. Status: {status_str}" use_this_sample = False if msg: _LOGGER.info(msg) diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py index 6be068c42..082afd3a9 100644 --- a/tests/smoketests/test_other.py +++ b/tests/smoketests/test_other.py @@ -61,6 +61,25 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd): raise pytest.fail("DID RAISE {0}".format(Exception)) +class TestLooperRerun: + @pytest.mark.parametrize( + "flags", [FLAGS[2], FLAGS[3]] + ) # Waiting and Failed flags should work + @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"]) + def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags): + """Verify that rerun works with either failed or waiting flags""" + tp = prep_temp_pep_pipestat + _make_flags(tp, flags, pipeline_name) + + x = ["rerun", "--looper-config", tp] + try: + result = main(test_args=x) + except Exception: + raise pytest.fail("DID RAISE {0}".format(Exception)) + + assert result["Jobs submitted"] == 2 + + class TestLooperCheck: @pytest.mark.parametrize("flag_id", FLAGS) @pytest.mark.parametrize(