From 6a66d3100c9761b4e94449aba0ea7c42ce0e23cb Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:28:38 -0400 Subject: [PATCH] refactor tests for new hello_looper changes --- tests/conftest.py | 10 ++--- tests/data/hello_looper-dev/.looper.yaml | 4 -- tests/data/hello_looper-dev/README.md | 12 +++-- tests/data/hello_looper-dev/csv/.looper.yaml | 5 --- .../hello_looper-dev/csv/data/frog1_data.txt | 4 -- .../hello_looper-dev/csv/data/frog2_data.txt | 7 --- .../csv/pipeline/pipeline_interface.yaml | 10 ----- .../csv/project/sample_annotation.csv | 3 -- .../input_schema_example/.looper.yaml | 4 ++ .../input_schema_example/data/canada.txt | 10 +++++ .../input_schema_example/data/mexico.txt | 31 +++++++++++++ .../input_schema_example/data/switzerland.txt | 23 ++++++++++ .../metadata/pep_config.yaml | 9 ++++ .../metadata/sample_table.csv | 5 +++ .../pipeline/count_lines.sh | 4 ++ .../pipeline/input_schema.yaml | 26 +++++++++++ .../pipeline/pipeline_interface.yaml | 10 +++++ .../pipeline/resources-sample.tsv | 4 ++ .../looper_csv_example/.looper.yaml | 4 ++ .../looper_csv_example/data/canada.txt | 10 +++++ .../looper_csv_example/data/mexico.txt | 31 +++++++++++++ .../looper_csv_example/data/switzerland.txt | 23 ++++++++++ .../metadata/sample_table.csv | 4 ++ .../pipeline/count_lines.sh | 2 +- .../pipeline/pipeline_interface.yaml | 4 ++ .../hello_looper-dev/minimal/.looper.yaml | 5 --- .../minimal/pipeline/pipeline_interface.yaml | 6 --- .../minimal/project/project_config.yaml | 2 - .../minimal/project/sample_annotation.csv | 3 -- .../pep_derived_attrs/.looper.yaml | 4 ++ .../pep_derived_attrs/data/canada.txt | 10 +++++ .../pep_derived_attrs/data/mexico.txt | 31 +++++++++++++ .../pep_derived_attrs/data/switzerland.txt | 23 ++++++++++ .../metadata/pep_config.yaml | 9 ++++ .../metadata/sample_table.csv | 4 ++ .../pipeline/count_lines.sh | 2 +- .../pipeline/count_lines_plot.py | 32 +++++++++++++ .../pipeline/pipeline_interface.yaml | 7 +++ .../data/hello_looper-dev/pephub/.looper.yaml | 2 +- .../hello_looper-dev/pephub/data/canada.txt | 10 +++++ .../pephub/data/frog1_data.txt | 4 -- .../pephub/data/frog2_data.txt | 7 --- .../hello_looper-dev/pephub/data/mexico.txt | 31 +++++++++++++ .../pephub/data/switzerland.txt | 23 ++++++++++ .../pephub/pipeline/pipeline_interface.yaml | 8 +--- .../hello_looper-dev/pipestat/data/frog_1.txt | 4 -- .../hello_looper-dev/pipestat/data/frog_2.txt | 7 --- .../pipestat_example/.looper.yaml | 8 ++++ .../.looper_pipestat_shell.yaml | 7 +++ .../pipestat_example/data/canada.txt | 10 +++++ .../pipestat_example/data/mexico.txt | 31 +++++++++++++ .../pipestat_example/data/switzerland.txt | 23 ++++++++++ .../pipestat_example/metadata/pep_config.yaml | 9 ++++ .../metadata/sample_table.csv | 4 ++ .../pipeline}/count_lines.py | 0 .../pipeline/count_lines.sh} | 0 .../pipeline/count_lines_plot.py | 45 +++++++++++++++++++ .../pipeline/pipeline_interface.yaml | 8 ++++ .../pipeline/pipeline_interface_shell.yaml | 5 +++ .../pipeline/pipestat_output_schema.yaml | 32 +++++++++++++ .../data/hello_looper-dev/pytesting/README.md | 1 + .../advanced_test}/.looper.yaml | 0 .../.looper_advanced_pipestat.yaml | 0 .../advanced_test}/pipeline/col_pipeline1.py | 0 .../advanced_test}/pipeline/col_pipeline2.py | 0 .../pipeline/other_pipeline2.py | 0 .../pipeline/output_schema.yaml | 0 .../advanced_test}/pipeline/pipeline1.py | 0 .../pipeline/pipeline_interface1_project.yaml | 0 .../pipeline/pipeline_interface1_sample.yaml | 0 .../pipeline/pipeline_interface2_project.yaml | 0 .../pipeline/pipeline_interface2_sample.yaml | 0 .../pipeline/pipestat_output_schema.yaml | 0 .../pipestat_pipeline_interface1_sample.yaml | 0 .../pipestat_pipeline_interface2_sample.yaml | 0 .../advanced_test}/pipeline/readData.R | 0 .../pipeline/resources-project.tsv | 0 .../pipeline/resources-sample.tsv | 0 .../project/annotation_sheet.csv | 0 .../project/project_config.yaml | 0 .../intermediate_test}/.looper.yaml | 0 .../intermediate_test}/data/frog_1.txt | 0 .../intermediate_test}/data/frog_2.txt | 0 .../pipeline/count_lines.sh | 0 .../pipeline/pipeline_interface.yaml | 0 .../project/project_config.yaml | 0 .../project/sample_annotation.csv | 0 .../pipestat_test}/.looper.yaml | 0 .../.looper_pipestat_shell.yaml | 0 .../pipestat_test}/data/frog_1.txt | 0 .../pipestat_test}/data/frog_2.txt | 0 .../pipeline_pipestat/count_lines.py | 31 +++++++++++++ .../pipeline_pipestat/count_lines_pipestat.sh | 4 ++ .../pipeline_pipestat/pipeline_interface.yaml | 0 .../pipeline_interface_shell.yaml | 0 .../pipestat_output_schema.yaml | 0 .../project/project_config.yaml | 0 .../project/sample_annotation.csv | 0 tests/smoketests/test_run.py | 2 +- 99 files changed, 619 insertions(+), 94 deletions(-) delete mode 100644 tests/data/hello_looper-dev/.looper.yaml delete mode 100644 tests/data/hello_looper-dev/csv/.looper.yaml delete mode 100644 tests/data/hello_looper-dev/csv/data/frog1_data.txt delete mode 100644 tests/data/hello_looper-dev/csv/data/frog2_data.txt delete mode 100644 tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml delete mode 100644 tests/data/hello_looper-dev/csv/project/sample_annotation.csv create mode 100644 tests/data/hello_looper-dev/input_schema_example/.looper.yaml create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/canada.txt create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/mexico.txt create mode 100644 tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt create mode 100644 tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml create mode 100644 tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv create mode 100755 tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml create mode 100644 tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv create mode 100644 tests/data/hello_looper-dev/looper_csv_example/.looper.yaml create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/canada.txt create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt create mode 100644 tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt create mode 100644 tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv rename tests/data/hello_looper-dev/{intermediate => looper_csv_example}/pipeline/count_lines.sh (69%) create mode 100644 tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml delete mode 100644 tests/data/hello_looper-dev/minimal/.looper.yaml delete mode 100644 tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml delete mode 100644 tests/data/hello_looper-dev/minimal/project/project_config.yaml delete mode 100644 tests/data/hello_looper-dev/minimal/project/sample_annotation.csv create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv rename tests/data/hello_looper-dev/{minimal => pep_derived_attrs}/pipeline/count_lines.sh (69%) create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py create mode 100644 tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml create mode 100644 tests/data/hello_looper-dev/pephub/data/canada.txt delete mode 100644 tests/data/hello_looper-dev/pephub/data/frog1_data.txt delete mode 100644 tests/data/hello_looper-dev/pephub/data/frog2_data.txt create mode 100644 tests/data/hello_looper-dev/pephub/data/mexico.txt create mode 100644 tests/data/hello_looper-dev/pephub/data/switzerland.txt delete mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_1.txt delete mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_2.txt create mode 100644 tests/data/hello_looper-dev/pipestat_example/.looper.yaml create mode 100644 tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/canada.txt create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/mexico.txt create mode 100644 tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt create mode 100644 tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml create mode 100644 tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv rename tests/data/hello_looper-dev/{pipestat/pipeline_pipestat => pipestat_example/pipeline}/count_lines.py (100%) rename tests/data/hello_looper-dev/{pipestat/pipeline_pipestat/count_lines_pipestat.sh => pipestat_example/pipeline/count_lines.sh} (100%) create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml create mode 100644 tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml create mode 100644 tests/data/hello_looper-dev/pytesting/README.md rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/.looper.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/.looper_advanced_pipestat.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/col_pipeline1.py (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/col_pipeline2.py (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/other_pipeline2.py (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/output_schema.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline1.py (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface1_project.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface1_sample.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface2_project.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipeline_interface2_sample.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_output_schema.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_pipeline_interface1_sample.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/pipestat_pipeline_interface2_sample.yaml (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/readData.R (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/resources-project.tsv (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/pipeline/resources-sample.tsv (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/project/annotation_sheet.csv (100%) rename tests/data/hello_looper-dev/{advanced => pytesting/advanced_test}/project/project_config.yaml (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/.looper.yaml (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/data/frog_1.txt (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/data/frog_2.txt (100%) rename tests/data/hello_looper-dev/{csv => pytesting/intermediate_test}/pipeline/count_lines.sh (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/pipeline/pipeline_interface.yaml (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/project/project_config.yaml (100%) rename tests/data/hello_looper-dev/{intermediate => pytesting/intermediate_test}/project/sample_annotation.csv (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/.looper.yaml (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/.looper_pipestat_shell.yaml (100%) rename tests/data/hello_looper-dev/{minimal => pytesting/pipestat_test}/data/frog_1.txt (100%) rename tests/data/hello_looper-dev/{minimal => pytesting/pipestat_test}/data/frog_2.txt (100%) create mode 100755 tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py create mode 100755 tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipeline_interface.yaml (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipeline_interface_shell.yaml (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/pipeline_pipestat/pipestat_output_schema.yaml (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/project/project_config.yaml (100%) rename tests/data/hello_looper-dev/{pipestat => pytesting/pipestat_test}/project/sample_annotation.csv (100%) diff --git a/tests/conftest.py b/tests/conftest.py index cfc104574..960a98b44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -204,7 +204,7 @@ def prep_temp_pep(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "advanced") + advanced_dir = os.path.join(d, "pytesting/advanced_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -220,7 +220,7 @@ def prep_temp_pep_basic(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "intermediate") + advanced_dir = os.path.join(d, "pytesting/intermediate_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -236,7 +236,7 @@ def prep_temp_pep_csv(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "csv") + advanced_dir = os.path.join(d, "looper_csv_example") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -274,7 +274,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "pipestat") + advanced_dir = os.path.join(d, "pytesting/pipestat_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -291,7 +291,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "advanced") + advanced_dir = os.path.join(d, "pytesting/advanced_test") path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml") return path_to_looper_config diff --git a/tests/data/hello_looper-dev/.looper.yaml b/tests/data/hello_looper-dev/.looper.yaml deleted file mode 100644 index e812a1ea8..000000000 --- a/tests/data/hello_looper-dev/.looper.yaml +++ /dev/null @@ -1,4 +0,0 @@ -pep_config: ./project/project_config.yaml # pephub registry path or local path -output_dir: "./results" -pipeline_interfaces: - sample: ../pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/README.md b/tests/data/hello_looper-dev/README.md index 6c213b1a9..a2a75feea 100644 --- a/tests/data/hello_looper-dev/README.md +++ b/tests/data/hello_looper-dev/README.md @@ -4,12 +4,10 @@ This repository provides minimal working examples for the [looper pipeline submi This repository contains examples -1. `/minimal` - A basic example pipeline and project. -2. `/intermediate` - An intermediate example pipeline and project with a couple extra options. -3. `/advanced` - A more advanced example, showcasing the capabilities of Looper. -4. `/pephub` - Example of how to point looper to PEPhub. -5. `/pipestat` - Example of a pipeline that uses pipestat for recording results. -6. `/csv` - How to use a pipeline with a CSV sample table (no YAML config) +1. `/looper_csv_example` - A minimal example using _only_ csv for metadata. +2. `/pep_derived_attributes` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata +3. `/pephub` - Example of how to point looper to a PEP stored on PEPhub and running a pipeline. +4. `/pipestat` - Example on how to use pipestat to report pipeline results when using looper. Each example contains: @@ -17,4 +15,4 @@ Each example contains: 2. Sample data plus metadata in PEP format (or pointer to PEPhub). 3. A looper-compatible pipeline. -Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/) +Explanation and results of running the above examples can be found at [Looper: Hello World Tutorial](https://pep.databio.org/looper/tutorial/initialize/) diff --git a/tests/data/hello_looper-dev/csv/.looper.yaml b/tests/data/hello_looper-dev/csv/.looper.yaml deleted file mode 100644 index 886b301c7..000000000 --- a/tests/data/hello_looper-dev/csv/.looper.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pep_config: project/sample_annotation.csv # local path to CSV -# pep_config: pepkit/hello_looper:default # you can also use a pephub registry path -output_dir: "results" -pipeline_interfaces: - - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/csv/data/frog1_data.txt b/tests/data/hello_looper-dev/csv/data/frog1_data.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/csv/data/frog1_data.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/csv/data/frog2_data.txt b/tests/data/hello_looper-dev/csv/data/frog2_data.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/csv/data/frog2_data.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml deleted file mode 100644 index 1b8a77eb8..000000000 --- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -sample_interface: - command_template: > - {pipeline.var_templates.pipeline} {sample.file} -project_interface: - command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv b/tests/data/hello_looper-dev/csv/project/sample_annotation.csv deleted file mode 100644 index 05bf4d172..000000000 --- a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv +++ /dev/null @@ -1,3 +0,0 @@ -sample_name,library,file,toggle -frog_1,anySampleType,data/frog1_data.txt,1 -frog_2,anySampleType,data/frog2_data.txt,1 diff --git a/tests/data/hello_looper-dev/input_schema_example/.looper.yaml b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml new file mode 100644 index 000000000..ba4eb61d0 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/pep_config.yaml +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/input_schema_example/data/canada.txt b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv new file mode 100644 index 000000000..dcbd2366d --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv @@ -0,0 +1,5 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province +usa, \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh new file mode 100755 index 000000000..03675d99c --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh @@ -0,0 +1,4 @@ +#!/bin/bash +linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` +export area_type=$2 +echo "Number of ${area_type}s: $linecount" diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml new file mode 100644 index 000000000..1de22ea3b --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml @@ -0,0 +1,26 @@ +description: An input schema for count_lines pipeline pipeline. +properties: + samples: + type: array + items: + type: object + properties: + sample_name: + type: string + description: "Name of the sample" + file_path: + type: string + description: "Path to the input file to count" + area_type: + type: string + description: "Name of the components of the country" + tangible: + - file_path + sizing: + - file_path + required: + - sample_name + - area_type + - file_path +required: + - samples diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..6f7b156bc --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,10 @@ +pipeline_name: count_lines +input_schema: input_schema.yaml +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} {sample.area_type} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/ +compute: + size_dependent_variables: resources-sample.tsv \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv new file mode 100644 index 000000000..6f0553ef6 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv @@ -0,0 +1,4 @@ +max_file_size cores mem time +0.0005 1 1000 00-01:00:00 +0.05 2 2000 00-03:00:00 +NaN 4 4000 00-05:00:00 diff --git a/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml new file mode 100644 index 000000000..381d1819a --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/sample_table.csv +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv new file mode 100644 index 000000000..6aa56d5bd --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type,file_path +mexico,state,data/mexico.txt +switzerland,canton,data/switzerland.txt +canada,province,data/canada.txt \ No newline at end of file diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh similarity index 69% rename from tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh index 71b887fe7..0a39a4a87 100755 --- a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh +++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh @@ -1,3 +1,3 @@ #!/bin/bash linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` -echo "Number of lines: $linecount" +echo "Number of lines: $linecount" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..dc58569b8 --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,4 @@ +pipeline_name: count_lines +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} diff --git a/tests/data/hello_looper-dev/minimal/.looper.yaml b/tests/data/hello_looper-dev/minimal/.looper.yaml deleted file mode 100644 index 4fcf56725..000000000 --- a/tests/data/hello_looper-dev/minimal/.looper.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pep_config: project/project_config.yaml # local path to pep config -# pep_config: pepkit/hello_looper:default # you can also use a pephub registry path -output_dir: "results" -pipeline_interfaces: - - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml deleted file mode 100644 index 58ddbf2b7..000000000 --- a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -sample_interface: - command_template: > - {pipeline.var_templates.pipeline} {sample.file} diff --git a/tests/data/hello_looper-dev/minimal/project/project_config.yaml b/tests/data/hello_looper-dev/minimal/project/project_config.yaml deleted file mode 100644 index 5456cca30..000000000 --- a/tests/data/hello_looper-dev/minimal/project/project_config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -pep_version: 2.0.0 -sample_table: sample_annotation.csv \ No newline at end of file diff --git a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv b/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv deleted file mode 100644 index 97f223700..000000000 --- a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv +++ /dev/null @@ -1,3 +0,0 @@ -sample_name,library,file,toggle -frog_1,anySampleType,data/frog_1.txt,1 -frog_2,anySampleType,data/frog_2.txt,1 diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml new file mode 100644 index 000000000..ba4eb61d0 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/pep_config.yaml +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv new file mode 100644 index 000000000..85f38a908 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province \ No newline at end of file diff --git a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh similarity index 69% rename from tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh index 71b887fe7..0a39a4a87 100755 --- a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh @@ -1,3 +1,3 @@ #!/bin/bash linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` -echo "Number of lines: $linecount" +echo "Number of lines: $linecount" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py new file mode 100644 index 000000000..398c1c02a --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py @@ -0,0 +1,32 @@ +import matplotlib.pyplot as plt +import os +import sys + +results_dir = sys.argv[ + 1 +] # Obtain the looper results directory passed via the looper command template + +# Extract the previously reported sample-level data from the .log files +countries = [] +number_of_regions = [] +for filename in os.listdir(results_dir): + if filename.endswith(".log"): + file = os.path.join(results_dir, filename) + with open(file, "r") as f: + for line in f: + if line.startswith("Number of lines:"): + region_count = int(line.split(":")[1].strip()) + number_of_regions.append(region_count) + country = filename.split("_")[2].split(".")[0] + countries.append(country) + +# Create a bar chart of regions per country +plt.figure(figsize=(8, 5)) +plt.bar(countries, number_of_regions, color=["blue", "green", "purple"]) +plt.xlabel("Countries") +plt.ylabel("Number of regions") +plt.title("Number of regions per country") + +# Save the image locally +save_location = os.path.join(os.path.dirname(results_dir), "regions_per_country.png") +plt.savefig(save_location, dpi=150) diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..dacc65891 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml @@ -0,0 +1,7 @@ +pipeline_name: count_lines +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/ diff --git a/tests/data/hello_looper-dev/pephub/.looper.yaml b/tests/data/hello_looper-dev/pephub/.looper.yaml index c2d74e0c2..654c5427e 100644 --- a/tests/data/hello_looper-dev/pephub/.looper.yaml +++ b/tests/data/hello_looper-dev/pephub/.looper.yaml @@ -1,4 +1,4 @@ -pep_config: pepkit/hello_looper:default # pephub registry path or local path +pep_config: databio/hello_looper:default output_dir: results pipeline_interfaces: - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/pephub/data/canada.txt b/tests/data/hello_looper-dev/pephub/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt b/tests/data/hello_looper-dev/pephub/data/frog1_data.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt b/tests/data/hello_looper-dev/pephub/data/frog2_data.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pephub/data/mexico.txt b/tests/data/hello_looper-dev/pephub/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pephub/data/switzerland.txt b/tests/data/hello_looper-dev/pephub/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml index 9bea7b43d..dc58569b8 100644 --- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml +++ b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml @@ -1,10 +1,4 @@ pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' sample_interface: command_template: > - {pipeline.var_templates.pipeline} {sample.file} -project_interface: - command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" + pipeline/count_lines.sh {sample.file_path} diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev/pipestat/data/frog_1.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev/pipestat/data/frog_2.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml new file mode 100644 index 000000000..dba1fbe9a --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml @@ -0,0 +1,8 @@ +pep_config: ./metadata/pep_config.yaml # pephub registry path or local path +output_dir: ./results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml +pipestat: + project_name: count_lines + results_file_path: results.yaml + flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml new file mode 100644 index 000000000..29a397194 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml @@ -0,0 +1,7 @@ +pep_config: ./metadata/pep_config.yaml # pephub registry path or local path +output_dir: ./results +pipeline_interfaces: + - pipeline/pipeline_interface_shell.yaml +pipestat: + results_file_path: results.yaml + flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/data/canada.txt b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv new file mode 100644 index 000000000..85f38a908 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py new file mode 100644 index 000000000..bc3a2bce3 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py @@ -0,0 +1,45 @@ +import matplotlib.pyplot as plt # be sure to `pip install matplotlib` +import os +import pipestat +import sys + +# A pipeline that retrieves previously reported pipestat results +# and plots them in a bar chart +results_file = sys.argv[1] +schema_path = sys.argv[2] + +# Create pipestat manager +psm = pipestat.PipestatManager( + schema_path=schema_path, results_file_path=results_file, pipeline_type="project" +) + +# Extract the previously reported data +results = ( + psm.select_records() +) # pipestat object holds the data after reading the results file +countries = [record["record_identifier"] for record in results["records"]] +number_of_regions = [record["number_of_lines"] for record in results["records"]] + +# Create a bar chart of regions per country +plt.figure(figsize=(8, 5)) +plt.bar(countries, number_of_regions, color=["blue", "green", "purple"]) +plt.xlabel("Countries") +plt.ylabel("Number of regions") +plt.title("Number of regions per country") +# plt.show() # Showing the figure and then saving it causes issues, so leave this commented out. + +# Save the image locally AND report that location via pipestat +# we can place it next to the results file for now +save_location = os.path.join(os.path.dirname(results_file), "regions_per_country.png") + +plt.savefig(save_location, dpi=150) + +result_to_report = { + "regions_plot": { + "path": save_location, + "thumbnail_path": save_location, + "title": "Regions Plot", + } +} + +psm.report(record_identifier="count_lines", values=result_to_report) diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..275af085a --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,8 @@ +pipeline_name: count_lines +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + python3 {looper.piface_dir}/count_lines.py {sample.file_path} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {pipestat.results_file} {pipestat.output_schema} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml new file mode 100644 index 000000000..c3b930fb3 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml @@ -0,0 +1,5 @@ +pipeline_name: count_lines +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + {looper.piface_dir}/count_lines.sh {sample.file_path} {sample.sample_name} {pipestat.config_file} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml new file mode 100644 index 000000000..7bedfd7e7 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml @@ -0,0 +1,32 @@ +title: Pipestat output schema for counting lines +description: A pipeline that uses pipestat to report sample level results. +type: object +properties: + pipeline_name: count_lines + samples: + type: array + items: + type: object + properties: + number_of_lines: + type: integer + description: "Number of lines in the input file." + project: + type: object + properties: + regions_plot: + description: "This a path to the output image" + image: + type: object + object_type: image + properties: + path: + type: string + thumbnail_path: + type: string + title: + type: string + required: + - path + - thumbnail_path + - title \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pytesting/README.md b/tests/data/hello_looper-dev/pytesting/README.md new file mode 100644 index 000000000..1f0c5cc7a --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/README.md @@ -0,0 +1 @@ +The examples in this pytesting directory are for Looper testing purposes. \ No newline at end of file diff --git a/tests/data/hello_looper-dev/advanced/.looper.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/.looper.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml diff --git a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/readData.R b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/readData.R rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv diff --git a/tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv b/tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv similarity index 100% rename from tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv diff --git a/tests/data/hello_looper-dev/advanced/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/intermediate/.looper.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml similarity index 100% rename from tests/data/hello_looper-dev/intermediate/.looper.yaml rename to tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt similarity index 100% rename from tests/data/hello_looper-dev/intermediate/data/frog_1.txt rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt similarity index 100% rename from tests/data/hello_looper-dev/intermediate/data/frog_2.txt rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt diff --git a/tests/data/hello_looper-dev/csv/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh similarity index 100% rename from tests/data/hello_looper-dev/csv/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml similarity index 100% rename from tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml rename to tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/intermediate/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/intermediate/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv similarity index 100% rename from tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv diff --git a/tests/data/hello_looper-dev/pipestat/.looper.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/.looper.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml diff --git a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml diff --git a/tests/data/hello_looper-dev/minimal/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt similarity index 100% rename from tests/data/hello_looper-dev/minimal/data/frog_1.txt rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt diff --git a/tests/data/hello_looper-dev/minimal/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt similarity index 100% rename from tests/data/hello_looper-dev/minimal/data/frog_2.txt rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py new file mode 100755 index 000000000..6f6a4ab8f --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py @@ -0,0 +1,31 @@ +import os.path + +import pipestat +import sys + +# Very simple pipeline that calls pipestat +# takes arguments invoked during looper submission via command templates +text_file = sys.argv[ + 1 +] # this is the sample we wish to process by reading the number of lines +sample_name = sys.argv[2] +results_file = sys.argv[3] +schema_path = sys.argv[4] + +# Create pipestat manager and then report values +psm = pipestat.PipestatManager( + schema_path=schema_path, + results_file_path=results_file, + record_identifier=sample_name, +) + + +text_file = os.path.abspath(text_file) +# Read text file and count lines +with open(text_file, "r") as f: + result = {"number_of_lines": len(f.readlines())} + +# The results are defined in the pipestat output schema. +psm.report(record_identifier=sample_name, values=result) + +# end of pipeline diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh new file mode 100755 index 000000000..99f83f906 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh @@ -0,0 +1,4 @@ +#!/bin/bash +linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` +pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3 +echo "Number of lines: $linecount" diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml diff --git a/tests/data/hello_looper-dev/pipestat/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv similarity index 100% rename from tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py index 59eede55f..c35d59470 100644 --- a/tests/smoketests/test_run.py +++ b/tests/smoketests/test_run.py @@ -634,4 +634,4 @@ def test_init_project_using_csv(self, prep_temp_pep_csv): pep_config_csv = os.path.join(os.path.dirname(tp), pep_config_csv) init_project = Project(cfg=pep_config_csv) - assert len(init_project.samples) == 2 + assert len(init_project.samples) == 3