Merge pull request #43 from fmalmeida/dev

Hotfix: v2.7.1
fmalmeida · Jun 20, 2024 · 5d16679 · 5d16679
2 parents cea2291 + 7d732a8
commit 5d16679
Show file tree

Hide file tree

Showing 12 changed files with 199 additions and 22 deletions.
diff --git a/.github/workflows/nftest.yml b/.github/workflows/nftest.yml
@@ -0,0 +1,77 @@
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-test CI
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
+  merge_group:
+    types:
+      - checks_requested
+    branches:
+      - master
+      - dev
+
+env:
+  NXF_ANSI_LOG: false
+  NFTEST_VER: "0.8.1"
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: "NF: ${{ matrix.NXF_VER }}"
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'fmalmeida/ngs-preprocess') }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        NXF_VER:
+          - "23.04.0"
+
+    steps:
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
+      - name: Check out pipeline code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Run nf-test
+        run: |
+          nf-test test tests/main.nf.test --junitxml=test.xml
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,5 @@ teste
 testing
 .vscode/settings.json
 docs/_build
+*.log
+.nf-test/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 The tracking for changes started in v2.2
 
+## v2.7.1
+
+* [[#36]](https://github.com/fmalmeida/ngs-preprocess/issues/36) - Include nf-tests to the pipeline.
+* [[#40]](https://github.com/fmalmeida/ngs-preprocess/issues/40) - Fix a problem when downloading PACBIO data.
+* Adjust short-reads workflow filter to allow analysis from SRA for platform key-values: illumina, bgiseq and dnbseq
+
 ## v2.7.0 -- [2024-Apr-30]
 
 * [[#34]](https://github.com/fmalmeida/ngs-preprocess/issues/34) - Included a new tool in the pipeline (`porechop ABI`). Removed `tracedir` parameter. Build new docker image.

diff --git a/conf/defaults.config b/conf/defaults.config
@@ -23,7 +23,7 @@
 params {
 
   // Sets output directory
-  output   = "output"
+  output  = null
 
   // inputs from SRA. A file containing on SRA ID per line.
   sra_ids = null

diff --git a/main.nf b/main.nf
@@ -156,7 +156,7 @@ workflow {
   .filter{ it != '' }
   .mix( 
     SRA_FETCH.out.fastqs
-    .filter{ it[1] =~ /illumina|bgiseq/ }
+    .filter{ it[1] =~ /illumina|bgiseq|dnbseq/ }
     .map{ 
       def meta             = [:]
       meta.id              = it[0]

diff --git a/modules/get_fastq.nf b/modules/get_fastq.nf
@@ -15,7 +15,6 @@ process GET_FASTQ {
   def sra_ids = "${sra_ids.replaceAll(~/\s/,'')}"
   """
   fasterq-dump \\
-    --include-technical \\
     --split-files \\
     --threads $task.cpus \\
     --outdir ./${sra_ids}_data \\
@@ -26,12 +25,25 @@ process GET_FASTQ {
   if [ \$(grep -ic "is PACBIO, please use fastq-dump instead" fasterq-dump.err) -eq 1 ]
   then
       fastq-dump \\
-        --split-files \\
+        --gzip \\
         --outdir ./${sra_ids}_data \\
         $sra_ids
   else
     echo "fasterq-dump error was:"
     cat fasterq-dump.err
   fi
+
+  # make sure they have right extension
+  for i in \$( find ./${sra_ids}_data -name "*.fq" ) ; do
+    mv \$i \${i%%.fq}.fastq ;
+  done
+  for i in \$( find ./${sra_ids}_data -name "*.fq.gz" ) ; do
+    mv \$i \${i%%.fq.gz}.fastq.gz ;
+  done
+
+  # make sure data is compressed
+  for i in \$( find ./${sra_ids}_data -name "*.fastq" ) ; do
+    gzip \$i ;
+  done
   """
 }
diff --git a/nextflow.config b/nextflow.config
@@ -37,6 +37,22 @@ try {
     System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
 }
 
+/*
+                                    Setting up NF profiles
+                            To use different profiles and executors
+            please read more at: https://www.nextflow.io/docs/latest/config.html#config-profiles
+ */
+profiles {
+
+  // load profiles
+  standard    { includeConfig 'conf/standard.config'    }
+  conda       { includeConfig 'conf/conda.config'       }
+  docker      { includeConfig 'conf/docker.config'      }
+  singularity { includeConfig 'conf/singularity.config' }
+  test        { includeConfig 'conf/test.config'        }
+
+}
+
 /*
 
 
@@ -66,22 +82,6 @@ report {
     file = "${params.output}/pipeline_info/ngs_preprocess_report_${trace_timestamp}.html"
 }
 
-/*
-                                    Setting up NF profiles
-                            To use different profiles and executors
-            please read more at: https://www.nextflow.io/docs/latest/config.html#config-profiles
- */
-profiles {
-
-  // load profiles
-  standard    { includeConfig 'conf/standard.config'    }
-  conda       { includeConfig 'conf/conda.config'       }
-  docker      { includeConfig 'conf/docker.config'      }
-  singularity { includeConfig 'conf/singularity.config' }
-  test        { includeConfig 'conf/test.config'        }
-
-}
-
 /*
                                     Adding manifest
 */
@@ -92,5 +92,5 @@ manifest {
   homePage        = "https://github.com/fmalmeida/ngs-preprocess"
   mainScript      = "main.nf"
   nextflowVersion = ">=21.10.0"
-  version         = "2.7.0"
+  version         = "2.7.1"
 }
diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -13,7 +13,6 @@
             "properties": {
                 "output": {
                     "type": "string",
-                    "default": "output",
                     "help_text": "Where to place results?",
                     "fa_icon": "fas fa-folder"
                 },

diff --git a/nf-test.config b/nf-test.config
@@ -0,0 +1,8 @@
+config {
+
+    testsDir   "tests"
+    workDir    ".nf-test"
+    configFile "tests/nextflow.config"
+    profile    "docker"
+
+}
diff --git a/tests/main.nf.test b/tests/main.nf.test
@@ -0,0 +1,39 @@
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Testing workflow - Should run without fails") {
+
+        when {
+            params {
+                sra_ids    = 'https://github.com/fmalmeida/test_datasets/raw/main/sra_ids.txt'
+                output     = "${outputDir}/test_output"
+                max_cpus   = 2
+                max_memory = '6.GB'
+                max_time   = '6.h'
+            }
+        }
+
+        then {
+
+            assertAll(
+                // All good?
+                {assert workflow.success},
+
+                // All there?
+                {assert path("${outputDir}/test_output").list().size() == 5},
+                {assert path("${outputDir}/test_output/preprocessing_outputs").list().size() == 3},
+
+                // Check if files are the same
+                {assert snapshot(
+                    workflow.trace,
+                    path( "${outputDir}/test_output/final_output" )
+                ).match()}
+            )
+
+        }
+
+    }
+
+}
diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap
@@ -0,0 +1,29 @@
+{
+    "Testing workflow - Should run without fails": {
+        "content": [
+            {
+                "tasksFailed": 0,
+                "tasksCount": 16,
+                "tasksSucceeded": 16
+            },
+            [
+                [
+                    "SRR19901041.filtered.fq.gz:md5,1d7a8e18db3b1f7ac0bfc1f5cf1daddf"
+                ],
+                [
+                    "ERR7339227.filtered.fq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "SRR9641620.filtered.fq.gz:md5,f6658663be49766459b123efa690cc42"
+                ],
+                [
+                    "ERR9850226_R1.preprocessed.fq.gz:md5,69f8bd550fbf99d81f4a110734f4803a",
+                    "ERR9850226_R2.preprocessed.fq.gz:md5,7543436f1614ae814337a9b123d26376"
+                ]
+            ]
+        ],
+        "meta": {
+            "nf-test": "0.8.4",
+            "nextflow": "23.10.1"
+        },
+        "timestamp": "2024-05-10T15:44:14.430345179"
+    }
+}
diff --git a/tests/nextflow.config b/tests/nextflow.config
@@ -0,0 +1,5 @@
+/*
+========================================================================================
+    Nextflow config file for running tests
+========================================================================================
+*/