Merge pull request #26 from icgc-argo/ega-download-wf@0.1.3

[release]
icgc-argo · Aug 19, 2022 · 6fcef40 · 6fcef40
2 parents 1577ee4 + 88a29e0
commit 6fcef40
Show file tree

Hide file tree

Showing 40 changed files with 1,287 additions and 61 deletions.
diff --git a/ega-download-wf/main.nf b/ega-download-wf/main.nf
@@ -18,10 +18,11 @@
 
   Authors:
     Edmund Su
+    Linda Xiang
 */
 
 nextflow.enable.dsl = 2
-version = '0.1.2'
+version = '0.1.3'
 
 // universal params go here, change default value as needed
 params.container = ""
@@ -32,7 +33,6 @@ params.mem = 1  // GB
 params.publish_dir = ""  // set to empty string will disable publishDir
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
 params.cleanup = true
 
 params.download_mode=""
@@ -48,22 +48,17 @@ params.c4gh_pass_phrase=""
 params.pyega3_ega_user=""
 params.pyega3_ega_pass=""
 
-include { downloadPyega3 } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/download-pyega3@0.1.1/main.nf' params([*:params, 'cleanup': false])
-include { downloadAspera } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/download-aspera@0.1.0/main.nf' params([*:params, 'cleanup': false])
-include { decryptAspera } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.0/main.nf' params([*:params, 'cleanup': false])
+include { downloadPyega3 } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/download-pyega3@0.1.3/main.nf' params([*:params, 'cleanup': false])
+include { downloadAspera } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/download-aspera@0.1.2/main.nf' params([*:params, 'cleanup': false])
+include { decryptAspera } from './wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/main.nf' params([*:params, 'cleanup': false])
 
 // please update workflow code as needed
 workflow EgaDownloadWf {
   take:  // update as needed
     download_mode
     file_info_tsv
-    ascp_scp_host
-    ascp_scp_user
-    aspera_scp_pass
-    pyega3_ega_user
-    pyega3_ega_pass
-    c4gh_secret_key
-    c4gh_pass_phrase
+    dependency
+
   main:  // update as needed
     Channel.fromPath(file_info_tsv).splitCsv(sep:'\t',header:true).map( row -> row.path).set{file_ch}
     Channel.fromPath(file_info_tsv).splitCsv(sep:'\t',header:true).map( row -> row.ega_file_id).set{id_ch}
@@ -72,22 +67,18 @@ workflow EgaDownloadWf {
       downloadAspera(
         file_ch,
         id_ch,
-        ascp_scp_host,
-        ascp_scp_user,
-        aspera_scp_pass)
+        dependency)
 
       decryptAspera(
         downloadAspera.out.output_file,
-        file(c4gh_secret_key),
-        c4gh_pass_phrase
+        file(params.c4gh_secret_key)
         )
 
       sequence_files=decryptAspera.out.output_files.collect()
     } else if (download_mode=='pyega3'){
       downloadPyega3(
         id_ch,
-        pyega3_ega_user,
-        pyega3_ega_pass
+        dependency
         )
 
       sequence_files=downloadPyega3.out.output_files.collect()
@@ -96,7 +87,6 @@ workflow EgaDownloadWf {
       exit 1
     }
 
-
   emit:  // update as needed
     sequence_files
 
@@ -109,12 +99,6 @@ workflow {
   EgaDownloadWf(
     params.download_mode,
     params.file_info_tsv,
-    params.ascp_scp_host,
-    params.ascp_scp_user,
-    params.aspera_scp_pass,
-    params.pyega3_ega_user,
-    params.pyega3_ega_pass,
-    params.c4gh_secret_key,
-    params.c4gh_pass_phrase
+    true
   )
 }
diff --git a/ega-download-wf/pkg.json b/ega-download-wf/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "ega-download-wf",
-    "version": "0.1.2",
+    "version": "0.1.3",
     "description": "Workflow wrapper for various methods to download data from EGA-archive",
     "main": "main.nf",
     "deprecated": false,
@@ -12,16 +12,20 @@
         "url": "https://github.com/icgc-argo/argo-data-submission.git"
     },
     "dependencies": [
-        "github.com/icgc-argo/argo-data-submission/download-aspera@0.1.0",
-        "github.com/icgc-argo/argo-data-submission/download-pyega3@0.1.2",
-        "github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.0"
+        "github.com/icgc-argo/argo-data-submission/download-aspera@0.1.2",
+        "github.com/icgc-argo/argo-data-submission/download-pyega3@0.1.3",
+        "github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1"
     ],
     "devDependencies": [],
     "contributors": [
         {
             "name": "Edmund Su",
             "email": "edmund.su@oicr.on.ca"
-        }
+        },
+        {
+          "name": "Linda Xiang",
+          "email": "linda.xiang@oicr.on.ca"
+      }
     ],
     "license": "GNU Affero General Public License v3",
     "bugReport": "https://github.com/icgc-argo/argo-data-submission/issues",

diff --git a/ega-download-wf/tests/checker.nf b/ega-download-wf/tests/checker.nf
@@ -18,15 +18,11 @@
 
   Authors:
     Edmund Su
-*/
-
-/*
- This is an auto-generated checker workflow to test the generated main template workflow, it's
- meant to illustrate how testing works. Please update to suit your own needs.
+    Linda Xiang
 */
 
 nextflow.enable.dsl = 2
-version = '0.1.2'
+version = '0.1.3'
 
 // universal params
 params.publish_dir = ""
@@ -35,14 +31,23 @@ params.container_registry = ""
 params.container_version = ""
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.expected_output = ""
-params.cleanup = false
+params.download_mode=""
+params.file_info_tsv="NO_FILE1"
 
-include { EgaDownloadWf } from '../main'
-// include section starts
-// include section ends
+params.ascp_scp_host=""
+params.ascp_scp_user=""
+params.aspera_scp_pass=""
+
+params.c4gh_secret_key="NO_FILE2"
+params.c4gh_pass_phrase=""
+
+params.pyega3_ega_user=""
+params.pyega3_ega_pass=""
+
+params.expected_output = ""
+params.cleanup = true
 
+include { EgaDownloadWf } from '../main' params([*:params, 'cleanup': false])
 
 process file_smart_diff {
   input:
@@ -72,24 +77,28 @@ process file_smart_diff {
 
 workflow checker {
   take:
-    input_file
+    download_mode
+    file_info_tsv
     expected_output
 
   main:
     EgaDownloadWf(
-      input_file
+      download_mode,
+      file_info_tsv,
+      true
     )
 
     file_smart_diff(
-      EgaDownloadWf.out.output_file,
+      EgaDownloadWf.out.sequence_files,
       expected_output
     )
 }
 
 
 workflow {
   checker(
-    file(params.input_file),
+    params.download_mode,
+    params.file_info_tsv,
     file(params.expected_output)
   )
 }
diff --git a/ega-download-wf/tests/input/aspera.tsv b/ega-download-wf/tests/input/aspera.tsv
@@ -0,0 +1 @@
+ega_file_id	path
diff --git a/ega-download-wf/tests/input/pyega3.tsv b/ega-download-wf/tests/input/pyega3.tsv
@@ -1,2 +1,2 @@
 ega_file_id	path
-EGAF00001770106	EGAF00001770106.bam
+EGAF00001770106	
diff --git a/ega-download-wf/tests/input/test_rg_3.bam b/ega-download-wf/tests/input/test_rg_3.bam
diff --git a/ega-download-wf/tests/local-test-job-aspera.json b/ega-download-wf/tests/local-test-job-aspera.json
@@ -1,9 +1,4 @@
 {
-    "file_info_tsv" : "tests/input/pyega3.tsv",
-    "pyega3_ega_user" : "ega-test-data@ebi.ac.uk",
-    "pyega3_ega_pass" : "egarocks",
-    "download_mode" : "aspera",
-    "ascp_scp_host" : "",
-    "ascp_scp_user" : "",
-    "aspera_scp_pass" : ""
+    "file_info_tsv" : "input/aspera.tsv",
+    "download_mode" : "aspera"
 }
diff --git a/ega-download-wf/tests/local-test-job-pyega.json b/ega-download-wf/tests/local-test-job-pyega.json
@@ -1,9 +1,6 @@
 {
-    "file_info_tsv" : "tests/input/pyega3.tsv",
+    "file_info_tsv" : "input/pyega3.tsv",
     "pyega3_ega_user" : "ega-test-data@ebi.ac.uk",
     "pyega3_ega_pass" : "egarocks",
-    "download_mode" : "pyega3",
-    "ascp_scp_host" : "",
-    "ascp_scp_user" : "",
-    "aspera_scp_pass" : ""
+    "download_mode" : "pyega3"
 }
diff --git a/wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/.dockerignore b/wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/.dockerignore
@@ -0,0 +1,5 @@
+.gitignore
+.nextflow*
+tests
+work
+outdir
diff --git a/wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/Dockerfile b/wfpr_modules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/Dockerfile
@@ -0,0 +1,33 @@
+FROM ubuntu:18.04
+
+LABEL org.opencontainers.image.source https://github.com/icgc-argo/argo-data-submission
+
+LABEL org.opencontainers.image.authors Edmund Su (edmund.su@oicr.on.ca)
+LABEL org.opencontainers.image.title ICGC ARGO Data Decrypt files
+
+RUN apt-get update && \
+    apt-get install -y python3-pip python3-dev software-properties-common curl python3-setuptools && \
+    apt-get install -y libz-dev pkg-config libtool m4 autotools-dev automake libncurses5-dev libbz2-dev liblzma-dev
+
+RUN pip3 install --upgrade pip &&\
+	pip3 install numpy crypt4gh
+
+RUN mkdir /tools
+
+ENV PATH="/tools:${PATH}"
+
+COPY *.py /tools/
+
+RUN groupadd -g 1000 ubuntu &&\
+    useradd -l -u 1000 -g ubuntu ubuntu &&\
+    install -d -m 0755 -o ubuntu -g ubuntu /home/ubuntu
+
+ENV PATH="/tools:${PATH}"
+
+COPY *.py /tools/
+
+ENTRYPOINT ["/usr/bin/env"]
+
+USER ubuntu
+
+CMD ["/bin/bash"]
diff --git a/...odules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/README.md b/...odules/github.com/icgc-argo/argo-data-submission/decrypt-aspera@0.1.1/README.md
@@ -0,0 +1,53 @@
+# Package decrypt-aspera
+
+
+Cryp4gh wrapper to decrypted files
+
+
+## Inputs
+
+File with the file suffix ".c4gh"
+
+
+## Outputs
+
+Decrypted file (same name without ".c4gh" suffix)
+Decrypted file w/ md5sum (same name with ".md5" suffix)
+
+## Usage
+
+#### Contents of Param-file
+```
+{
+	"file":"input/mystery_file.bam.c4gh", ### File to decrypt
+	"c4gh_secret_key":"input/C4GH_SECRET_KEY.txt", ### Public secret key
+	"c4gh_pass_phrase":"" ### Public secret key passphrase
+}
+```
+
+#### Test run
+`nextflow run checker.nf -params-file test-job-decrypt.json`
+
+#### IRL run
+```
+nextflow run main.nf -params-file tests/test-job-decrypt.json
+```
+### Run the package directly
+
+
+With inputs prepared, you should be able to run the package directly using the following command.
+Please replace the params file with a real one (with all required parameters and input files). Example
+params file(s) can be found in the `tests` folder.
+
+```
+nextflow run edsu7/argo-data-submission/decrypt-aspera/main.nf -r decrypt-aspera.v0.1.0 -params-file <your-params-json-file>
+```
+
+### Import the package as a dependency
+
+To import this package into another package as a dependency, please follow these steps at the
+importing package side:
+
+1. add this package's URI `github.com/edsu7/argo-data-submission/decrypt-aspera@0.1.0` in the `dependencies` list of the `pkg.json` file
+2. run `wfpm install` to install the dependency
+3. add the `include` statement in the main Nextflow script to import the dependent package from this path: `./wfpr_modules/github.com/edsu7/argo-data-submission/decrypt-aspera@0.1.0/main.nf`