Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Consolidate pifaces #507

Merged
merged 13 commits into from
Jul 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions looper/cli_pydantic.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,11 @@
read_yaml_file,
inspect_looper_config_file,
is_PEP_file_type,
looper_config_tutorial,
)

from typing import List, Tuple
from rich.console import Console


def opt_attr_pair(name: str) -> Tuple[str, str]:
Expand Down Expand Up @@ -122,17 +124,34 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
sys.exit(1)

if subcommand_name == "init":
return int(
not initiate_looper_config(
dotfile_path(),
subcommand_args.pep_config,
subcommand_args.output_dir,
subcommand_args.sample_pipeline_interfaces,
subcommand_args.project_pipeline_interfaces,
subcommand_args.force_yes,
)

console = Console()
console.clear()
console.rule(f"\n[magenta]Looper initialization[/magenta]")
console.print(
"[bold]Would you like to follow a guided tutorial?[/bold] [green]Y[/green] / [red]n[/red]..."
)

selection = None
while selection not in ["y", "n"]:
selection = console.input("\nSelection: ").lower().strip()

if selection == "n":
console.clear()
return int(
not initiate_looper_config(
dotfile_path(),
subcommand_args.pep_config,
subcommand_args.output_dir,
subcommand_args.sample_pipeline_interfaces,
subcommand_args.project_pipeline_interfaces,
subcommand_args.force_yes,
)
)
else:
console.clear()
return int(looper_config_tutorial())

if subcommand_name == "init_piface":
sys.exit(int(not init_generic_pipeline()))

Expand Down Expand Up @@ -243,7 +262,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
# Check at the beginning if user wants to use pipestat and pipestat is configurable
is_pipestat_configured = (
prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
if getattr(subcommand_args, "project", None)
if getattr(subcommand_args, "project", None) or subcommand_name == "runp"
else prj._check_if_pipestat_configured()
)

Expand Down
9 changes: 8 additions & 1 deletion looper/conductor.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ def __init__(

self.collate = collate
self.section_key = PROJECT_PL_KEY if self.collate else SAMPLE_PL_KEY
self.pipeline_interface_type = (
"project_interface" if self.collate else "sample_interface"
)
self.pl_iface = pipeline_interface
self.pl_name = self.pl_iface.pipeline_name
self.prj = prj
Expand Down Expand Up @@ -681,7 +684,11 @@ def write_script(self, pool, size):
pipeline=self.pl_iface,
compute=self.prj.dcc.compute,
)
templ = self.pl_iface["command_template"]

if self.pipeline_interface_type is None:
templ = self.pl_iface["command_template"]
else:
templ = self.pl_iface[self.pipeline_interface_type]["command_template"]
if not self.override_extra:
extras_template = (
EXTRA_PROJECT_CMD_TEMPLATE
Expand Down
9 changes: 0 additions & 9 deletions looper/pipeline_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,6 @@ def __init__(self, config, pipeline_type=None):
)
self.update(config)
self._validate(schema_src=PIFACE_SCHEMA_SRC)
if "path" in self:
warn(
message="'path' specification as a top-level pipeline "
"interface key is deprecated and will be removed with "
"the next release. Please use 'paths' section "
"from now on.",
category=DeprecationWarning,
)
self._expand_paths(["path"])
self._expand_paths(["compute", "dynamic_variables_script_path"])

@property
Expand Down
16 changes: 9 additions & 7 deletions looper/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,10 +413,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value):
pipestat_config_path = self._check_for_existing_pipestat_config(piface)

if not pipestat_config_path:
self._create_pipestat_config(piface)
self._create_pipestat_config(piface, pipeline_type)
else:
piface.psm = PipestatManager(
config_file=pipestat_config_path, multi_pipelines=True
config_file=pipestat_config_path,
multi_pipelines=True,
pipeline_type="sample",
)

elif pipeline_type == PipelineLevel.PROJECT.value:
Expand All @@ -426,10 +428,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value):
)

if not pipestat_config_path:
self._create_pipestat_config(prj_piface)
self._create_pipestat_config(prj_piface, pipeline_type)
else:
prj_piface.psm = PipestatManager(
config_file=pipestat_config_path, multi_pipelines=True
config_file=pipestat_config_path,
multi_pipelines=True,
pipeline_type="project",
)
else:
_LOGGER.error(
Expand Down Expand Up @@ -469,7 +473,7 @@ def _check_for_existing_pipestat_config(self, piface):
else:
return None

def _create_pipestat_config(self, piface):
def _create_pipestat_config(self, piface, pipeline_type):
"""
Each piface needs its own config file and associated psm
"""
Expand Down Expand Up @@ -512,8 +516,6 @@ def _create_pipestat_config(self, piface):
pipestat_config_dict.update({"pipeline_name": piface.data["pipeline_name"]})
else:
pipeline_name = None
if "pipeline_type" in piface.data:
pipestat_config_dict.update({"pipeline_type": piface.data["pipeline_type"]})

# Warn user if there is a mismatch in pipeline_names from sources!!!
if pipeline_name != output_schema_pipeline_name:
Expand Down
20 changes: 14 additions & 6 deletions looper/schemas/pipeline_interface_schema_generic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,20 @@ properties:
type: string
enum: ["project", "sample"]
description: "type of the pipeline, either 'project' or 'sample'"
command_template:
type: string
description: "Jinja2-like template to construct the command to run"
path:
type: string
description: "path to the pipeline program. Relative to pipeline interface file or absolute."
sample_interface:
type: object
description: "Section that defines compute environment settings"
properties:
command_template:
type: string
description: "Jinja2-like template to construct the command to run"
project_interface:
type: object
description: "Section that defines compute environment settings"
properties:
command_template:
type: string
description: "Jinja2-like template to construct the command to run"
compute:
type: object
description: "Section that defines compute environment settings"
Expand Down
Loading
Loading