refactor tests for new hello_looper changes

tests/conftest.py

@@ -204,7 +204,7 @@ def prep_temp_pep(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "advanced")
+    advanced_dir = os.path.join(d, "pytesting/advanced_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -220,7 +220,7 @@ def prep_temp_pep_basic(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "intermediate")
+    advanced_dir = os.path.join(d, "pytesting/intermediate_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -236,7 +236,7 @@ def prep_temp_pep_csv(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "csv")
+    advanced_dir = os.path.join(d, "looper_csv_example")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -274,7 +274,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "pipestat")
+    advanced_dir = os.path.join(d, "pytesting/pipestat_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -291,7 +291,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "advanced")
+    advanced_dir = os.path.join(d, "pytesting/advanced_test")
     path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml")
 
     return path_to_looper_config

tests/data/hello_looper-dev/README.md

@@ -4,17 +4,15 @@ This repository provides minimal working examples for the [looper pipeline submi
 
 This repository contains examples
 
-1. `/minimal` - A basic example pipeline and project.
-2. `/intermediate` - An intermediate example pipeline and project with a couple extra options.
-3. `/advanced` - A more advanced example, showcasing the capabilities of Looper.
-4. `/pephub` - Example of how to point looper to PEPhub.
-5. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
-6. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
+1. `/looper_csv_example` - A minimal example using _only_ csv for metadata.
+2. `/pep_derived_attributes` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata
+3. `/pephub` - Example of how to point looper to a PEP stored on PEPhub and running a pipeline.
+4. `/pipestat` - Example on how to use pipestat to report pipeline results when using looper.
 
 Each example contains:
 
 1. A looper config file (`.looper.yaml`).
 2. Sample data plus metadata in PEP format (or pointer to PEPhub).
 3. A looper-compatible pipeline.
 
-Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/)
+Explanation and results of running the above examples can be found at [Looper: Hello World Tutorial](https://pep.databio.org/looper/tutorial/initialize/)

tests/data/hello_looper-dev/input_schema_example/.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: metadata/pep_config.yaml
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev/input_schema_example/data/canada.txt

@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan

tests/data/hello_looper-dev/input_schema_example/data/mexico.txt

@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas

tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt

@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura

tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml

@@ -0,0 +1,9 @@
+pep_version: 2.1.0
+sample_table: sample_table.csv
+sample_modifiers:
+  append:
+    file_path: source1
+  derive:
+    attributes: [file_path]
+    sources:
+      source1: "data/{sample_name}.txt"

tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv

@@ -0,0 +1,5 @@
+sample_name,area_type
+mexico,state
+switzerland,canton
+canada,province
+usa,

tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+export area_type=$2
+echo "Number of ${area_type}s: $linecount"

tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml

@@ -0,0 +1,26 @@
+description: An input schema for count_lines pipeline pipeline.
+properties:
+  samples:
+    type: array
+    items:
+      type: object
+      properties:
+        sample_name: 
+          type: string
+          description: "Name of the sample"
+        file_path: 
+          type: string
+          description: "Path to the input file to count"
+        area_type:
+          type: string
+          description: "Name of the components of the country"
+      tangible:
+        - file_path
+      sizing:
+        - file_path
+      required:
+        - sample_name
+        - area_type
+        - file_path
+required:
+  - samples

tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml

@@ -0,0 +1,10 @@
+pipeline_name: count_lines
+input_schema: input_schema.yaml
+sample_interface:
+  command_template: >
+    pipeline/count_lines.sh {sample.file_path} {sample.area_type}
+project_interface:
+  command_template: >
+    python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/
+compute:
+  size_dependent_variables: resources-sample.tsv

tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv

@@ -0,0 +1,4 @@
+max_file_size	cores	mem	time
+0.0005	1	1000	00-01:00:00
+0.05	2	2000	00-03:00:00
+NaN	4	4000	00-05:00:00

tests/data/hello_looper-dev/looper_csv_example/.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: metadata/sample_table.csv
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev/looper_csv_example/data/canada.txt

@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan

tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt

@@ -0,0 +1,31 @@
+Aguascalientes
+Baja California
+Baja California Sur
+Campeche
+Chiapas
+Chihuahua
+Coahuila
+Colima
+Durango
+Guanajuato
+Guerrero
+Hidalgo
+Jalisco
+México (Estado de México)
+Michoacán
+Morelos
+Nayarit
+Nuevo León
+Oaxaca
+Puebla
+Querétaro
+Quintana Roo
+San Luis Potosí
+Sinaloa
+Sonora
+Tabasco
+Tamaulipas
+Tlaxcala
+Veracruz
+Yucatán
+Zacatecas

tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt

@@ -0,0 +1,23 @@
+Zürich
+Bern
+Luzern
+Uri
+Schwyz
+Unterwalden
+Glarus
+Zug
+Freiburg
+Solothurn
+Basel
+Schaffhausen
+Appenzell
+Sankt Gallen
+Graubünden
+Aargau
+Thurgau
+Ticino
+Vaud
+Valais
+Neuchâtel
+Genève
+Jura

tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv

@@ -0,0 +1,4 @@
+sample_name,area_type,file_path
+mexico,state,data/mexico.txt
+switzerland,canton,data/switzerland.txt
+canada,province,data/canada.txt

tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh → tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
-echo "Number of lines: $linecount"
+echo "Number of lines: $linecount"

tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml

@@ -0,0 +1,4 @@
+pipeline_name: count_lines
+sample_interface:
+  command_template: >
+    pipeline/count_lines.sh {sample.file_path}

tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: metadata/pep_config.yaml
+output_dir: results
+pipeline_interfaces:
+  - pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt

@@ -0,0 +1,10 @@
+Alberta
+British Columbia
+Manitoba
+New Brunswick
+Newfoundland and Labrador
+Nova Scotia
+Ontario
+Prince Edward Island
+Quebec
+Saskatchewan