update advanced and pipestat examples, fix corresponding tests, add o…

…utput_schema to pipestat namespace

looper/conductor.py

@@ -429,11 +429,11 @@ def submit(self, force=False):
                 submission_command = "{} {}".format(sub_cmd, script)
                 # Capture submission command return value so that we can
                 # intercept and report basic submission failures; #167
-
                 process = subprocess.Popen(submission_command, stderr=PIPE, shell=True)
                 self.process_id = process.pid
                 output, errors = process.communicate()
-                if errors:
+                _LOGGER.debug(msg=errors)
+                if process.returncode != 0:
                     fails = (
                         "" if self.collate else [s.sample_name for s in self._samples]
                     )
@@ -654,6 +654,7 @@ def _set_pipestat_namespace(
                 "results_file": psm.file,
                 "record_identifier": psm.record_identifier,
                 "config_file": psm.config_path,
+                "output_schema": psm.cfg["_schema_path"],
             }
             filtered_namespace = {k: v for k, v in full_namespace.items() if v}
             return YAMLConfigManager(filtered_namespace)

tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml

@@ -2,10 +2,8 @@ pipeline_name: PIPELINE1
 pipeline_type: project
 output_schema: output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/col_pipeline1.py"
+  path: "{looper.piface_dir}/col_pipeline1.py"
 command_template: >
-  {pipeline.var_templates.path} --project-name {project.name}
+  python3 {pipeline.var_templates.path} --project-name {project.name}
+
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R

tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml

@@ -3,13 +3,11 @@ pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+  path: "{looper.piface_dir}/pipeline1.py"
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
 command_template: >
-  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R

tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml

@@ -2,12 +2,10 @@ pipeline_name: OTHER_PIPELINE2
 pipeline_type: project
 output_schema: output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/col_pipeline2.py"
+  path: "{looper.piface_dir}/col_pipeline2.py"
 command_template: >
-  {pipeline.var_templates.path} --project-name {project.name}
+  python3 {pipeline.var_templates.path} --project-name {project.name}
 compute:
   size_dependent_variables: resources-project.tsv
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R
+

tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml

@@ -2,15 +2,12 @@ pipeline_name: OTHER_PIPELINE2
 pipeline_type: sample
 output_schema: output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+  path: "{looper.piface_dir}/other_pipeline2.py"
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
 command_template: >
-  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 compute:
   size_dependent_variables: resources-sample.tsv
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R

tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml

@@ -3,13 +3,11 @@ pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: pipestat_output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+  path: "{looper.piface_dir}/pipeline1.py"
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
 command_template: >
-  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R

tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml

@@ -3,15 +3,13 @@ pipeline_type: sample
 input_schema: https://schema.databio.org/pep/2.0.0.yaml
 output_schema: pipestat_output_schema.yaml
 var_templates:
-  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+  path: "{looper.piface_dir}/other_pipeline2.py"
 pre_submit:
     python_functions:
       - looper.write_sample_yaml
 command_template: >
-  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+  python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
 compute:
   size_dependent_variables: resources-sample.tsv
 
-bioconductor:
-  readFunName: readData
-  readFunPath: readData.R
+

tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py

@@ -1,3 +1,5 @@
+import os.path
+
 import pipestat
 import sys
 
@@ -8,14 +10,17 @@
 ]  # this is the sample we wish to process by reading the number of lines
 sample_name = sys.argv[2]
 results_file = sys.argv[3]
+schema_path = sys.argv[4]
 
 # Create pipestat manager and then report values
 psm = pipestat.PipestatManager(
-    schema_path="pipeline_pipestat/pipestat_output_schema.yaml",
+    schema_path=schema_path,
     results_file_path=results_file,
     record_identifier=sample_name,
 )
 
+
+text_file = os.path.abspath(text_file)
 # Read text file and count lines
 with open(text_file, "r") as f:
     result = {"number_of_lines": len(f.readlines())}

tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml

@@ -2,4 +2,4 @@ pipeline_name: example_pipestat_pipeline
 pipeline_type: sample
 output_schema: pipestat_output_schema.yaml
 command_template: >
-  python {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file}
+  python {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema}

tests/smoketests/test_other.py

@@ -95,6 +95,30 @@ def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags):
         """Verify that rerun works with either failed or waiting flags"""
         tp = prep_temp_pep_pipestat
         _make_flags_pipestat(tp, flags, pipeline_name)
+        path_to_looper_config = prep_temp_pep_pipestat
+        pipestat_dir = os.path.dirname(path_to_looper_config)
+
+        # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+        pipestat_project_file = get_project_config_path(path_to_looper_config)
+
+        pipestat_pipeline_interface_file = os.path.join(
+            pipestat_dir, "pipeline_pipestat/pipeline_interface.yaml"
+        )
+
+        with open(pipestat_project_file, "r") as f:
+            pipestat_project_data = safe_load(f)
+
+        pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+            os.path.join(pipestat_dir, "data/{sample_name}.txt")
+        )
+
+        with open(pipestat_pipeline_interface_file, "r") as f:
+            pipestat_piface_data = safe_load(f)
+
+        pipeline_name = pipestat_piface_data["pipeline_name"]
+
+        with open(pipestat_project_file, "w") as f:
+            dump(pipestat_project_data, f)
 
         x = ["rerun", "--looper-config", tp]
         try:

tests/test_comprehensive.py

@@ -167,6 +167,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     tsv_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".tsv")]
     assert len(tsv_list) == 0
     with pytest.raises(RecordNotFoundError):
+        psm = PipestatManager(config_file=path_to_pipestat_config)
         retrieved_result = psm.retrieve_one(record_identifier="frog_2")