diff --git a/.editorconfig b/.editorconfig index b78de6e6..b6b31907 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js,cff}] +[*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 12cd5bb5..1e3f7273 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -45,6 +45,6 @@ body: * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ - * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ * Version of nf-core/methylseq _(eg. 1.1, 1.5, 1.8.2)_ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9dc0985d..08cee127 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,8 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/meth - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! -- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md) +- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 0527b4be..da863783 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -17,7 +17,7 @@ jobs: aligner: ["bismark", "bismark_hisat", "bwameth"] steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index b7d8c04f..478b2150 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -15,7 +15,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 352a5cf7..32d20557 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == 'nf-core/methylseq' run: | - { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7652b0f8..f0f49267 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,15 +1,20 @@ -name: nf-core CI # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +name: nf-core CI on: push: branches: - - dev + - "dev" pull_request: + branches: + - "dev" + - "master" release: - types: [published] + types: + - "published" env: NXF_ANSI_LOG: false + NFTEST_VER: "0.7.3" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -17,40 +22,57 @@ concurrency: jobs: test: - name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }} - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + name: Run pipeline with test data runs-on: ubuntu-latest strategy: + fail-fast: false matrix: NXF_VER: - "22.10.1" - "latest-everything" aligner: - - bismark - - bismark_hisat - - bwameth - profile: ["docker"] # TODO , "singularity", "conda"] + - "bismark" + - "bismark_hisat" + - "bwameth" + profile: + - "docker" steps: + # Skip if it's a pull_request to dev and NXF_VER is '22.10.1' + - name: Skip condition + id: condition + run: | + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then + echo "skip=true" >> $GITHUB_OUTPUT + fi + - name: Check out pipeline code + if: steps.condition.outputs.skip != 'true' uses: actions/checkout@v3 + - name: Hash Github Workspace + if: steps.condition.outputs.skip != 'true' + id: hash_workspace + run: | + echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data + if: steps.condition.outputs.skip != 'true' id: cache-testdata uses: actions/cache@v3 with: - path: ./test-datasets/ - key: methylseq-test-data + path: test-datasets/ + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data - if: steps.cache-testdata.outputs.cache-hit != 'true' + if: steps.cache-testdata.outputs.cache-hit != 'true' && steps.condition.outputs.skip != 'true' uses: actions/checkout@v3 with: repository: nf-core/test-datasets ref: methylseq - path: ./test-datasets/ + path: test-datasets/ - name: Replace remote paths in samplesheets + if: steps.condition.outputs.skip != 'true' run: | for f in ./test-datasets/samplesheet/*csv; do sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f @@ -61,41 +83,25 @@ jobs: done; - name: Install Nextflow + if: steps.condition.outputs.skip != 'true' uses: nf-core/setup-nextflow@v1 with: version: "${{ matrix.NXF_VER }}" - name: Install nf-test + if: steps.condition.outputs.skip != 'true' run: | - wget -qO- https://code.askimed.com/install/nf-test | bash + wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER sudo mv nf-test /usr/local/bin/ - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a - - name: Run nf-test + if: steps.condition.outputs.skip != 'true' run: | - nf-test test \ - --profile=test_ci,test,${{ matrix.profile }} \ - tests/${{ matrix.aligner }}/*.nf.test \ - --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap - - uses: pcolby/tap-summary@v1 - with: - path: >- - test.tap + # If the test fails, output the software_versions.yml using the 'batcat' utility + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml new file mode 100644 index 00000000..694e90ec --- /dev/null +++ b/.github/workflows/clean-up.yml @@ -0,0 +1,24 @@ +name: "Close user-tagged issues and PRs" +on: + schedule: + - cron: "0 0 * * 0" # Once a week + +jobs: + clean-up: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." + stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." + close-issue-message: "This issue was closed because it has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor and then staled for 20 days with no activity." + days-before-stale: 30 + days-before-close: 20 + days-before-pr-close: -1 + any-of-labels: "awaiting-changes,awaiting-feedback" + exempt-issue-labels: "WIP" + exempt-pr-labels: "WIP" + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 0b7d9bf2..a2f7f450 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 858d622e..888cb4bc 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.8" architecture: "x64" - name: Install dependencies diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 39635186..0bbcd30f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/.nf-core.yml b/.nf-core.yml index 83758400..798bca42 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,5 +1,8 @@ # FIXME - CI currently runs for dev and dsl2 branches, remove the latter before merging to dev -actions_ci: False -files_exist: - - bin/markdown_to_html.py repository_type: pipeline +lint: + files_unchanged: + - .gitattributes + files_exist: + - bin/markdown_to_html.py + actions_ci: False diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..0c31cdb9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier diff --git a/CHANGELOG.md b/CHANGELOG.md index 6149abd9..696d7219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # nf-core/methylseq +## [v2.4.0](https://github.com/nf-core/methylseq/releases/tag/2.4.0) - 2023-06-02 + +### Pipeline Updates + +- Updated template to nf-core/tools v2.8 +- Add `--bamqc_regions_file` parameter for targeted methylation sequencing data #302 +- ✨ Add NF-TEST tests and snapshots for the pipeline test profile #310 + +### Bug fixes & refactoring + +- 🛠️ update index file channels to explicit value channels #310 +- 🐛 fix `params.test_data_base` in test and test_full configs #310 +- 🤖 GitHub Actions CI - pull_request to `dev` tests with NXF_VER `latest-everything` #310 +- 🤖 GitHub Actions CI - pull_request to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 +- 🤖 GitHub Actions CI - `fail-fast` set to false #310 +- 🐛 get to the bottom of index tests #278 +- ✨ Support for Bismark methylation extraction `ignore` and `ignore_3prime` parameters when `ignore_r1` or `ignore_3prime_r1` are greater than 0. #322 +- 🛠️ rename `ignore` -> `ignore_r1` and `ignore_3prime` -> `ignore_3prime_r1` params #322 +- 🐛 fix `ignore_3prime_r2` param #299 +- 🐛 removed unused directory #297 + ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 ### Pipeline Updates diff --git a/README.md b/README.md index f9efb22e..8e039f1d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/methylseq) [![nf-test](https://img.shields.io/badge/tested_with-nf--test-337ab7.svg)](https://github.com/askimed/nf-test) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23methylseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/methylseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23methylseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/methylseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction @@ -40,31 +40,45 @@ Choose between workflows by using `--aligner bismark` (default, uses bowtie2 for | Sample complexity | Preseq | Preseq | | Project Report | MultiQC | MultiQC | -## Quick Start +## Usage -1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`) +> **Note** +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how +> to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) +> with `-profile test` before running the workflow on actual data. -2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_. +First, prepare a samplesheet with your input data that looks as follows: -3. Download the pipeline and test it on a minimal dataset with a single command: +`samplesheet.csv`: - ```bash - nextflow run nf-core/methylseq -profile test,YOURPROFILE --outdir - ``` +```csv +sample,fastq_1,fastq_2 +SRR389222_sub1,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz +SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz +SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub3.fastq.gz +Ecoli_10K_methylated,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz +``` -- Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. -- If you are using `singularity` then the pipeline will auto-detect this and attempt to download the Singularity images directly as opposed to performing a conversion from Docker images. If you are persistently observing issues downloading Singularity images directly due to timeout or network issues then please use the `--singularity_pull_docker_container` parameter to pull and convert the Docker image instead. It is also highly recommended to use the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) settings to store the images in a central location for future pipeline runs. -- If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. +Each row represents a fastq file (single-end) or a pair of fastq files (paired end). -4. Start running your own analysis! +Now, you can run the pipeline using: - ```bash - nextflow run nf-core/methylseq --input samplesheet.csv --outdir --genome GRCh37 -profile - ``` +```bash +nextflow run nf-core/methylseq --input samplesheet.csv --outdir --genome GRCh37 -profile +``` -## Documentation +> **Warning:** +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those +> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). -The nf-core/methylseq pipeline comes with documentation about the pipeline: [usage](https://nf-co.re/methylseq/usage) and [output](https://nf-co.re/methylseq/output). +For more details, please refer to the [usage documentation](https://nf-co.re/methylseq/usage) and the [parameter documentation](https://nf-co.re/methylseq/parameters). + +## Pipeline output + +To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/methylseq/results) tab on the nf-core website pipeline page. +For more details about the output files and reports, please refer to the +[output documentation](https://nf-co.re/methylseq/output). ## Credits diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 3d0a0c04..da1e666f 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -158,13 +158,7 @@ def sniff_format(handle): peek = read_head(handle) handle.seek(0) sniffer = csv.Sniffer() - try: - dialect = sniffer.sniff(peek) - except csv.Error as e: - logger.critical(f"Could not determine sample sheet delimiter: {handle.name}") - logger.critical(e) - sys.exit(1) - + dialect = sniffer.sniff(peek) return dialect diff --git a/conf/base.config b/conf/base.config index 8590a90b..e19bc5bb 100644 --- a/conf/base.config +++ b/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/conf/igenomes.config b/conf/igenomes.config index a1bfdabc..8442ed57 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -36,6 +36,14 @@ params { macs_gsize = "2.7e9" blacklist = "${projectDir}/assets/blacklists/hg38-blacklist.bed" } + 'CHM13' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAIndex/" + bwamem2 = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAmem2Index/" + gtf = "${params.igenomes_base}/Homo_sapiens/NCBI/CHM13/Annotation/Genes/genes.gtf" + gff = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/009/914/755/GCF_009914755.1_T2T-CHM13v2.0/GCF_009914755.1_T2T-CHM13v2.0_genomic.gff.gz" + mito_name = "chrM" + } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" diff --git a/conf/modules.config b/conf/modules.config index f9c155ae..50017087 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -209,9 +209,11 @@ process { params.comprehensive ? ' --comprehensive --merge_non_CpG' : '', params.meth_cutoff ? " --cutoff ${params.meth_cutoff}" : '', params.nomeseq ? '--CX' : '', + params.ignore_r1 > 0 ? '--ignore ${params.ignore_r1}' : '', + params.ignore_3prime_r1 > 0 ? '--ignore_3prime ${params.ignore_3prime_r1}' : '', meta.single_end ? '' : (params.no_overlap ? ' --no_overlap' : '--include_overlap'), meta.single_end ? '' : (params.ignore_r2 > 0 ? "--ignore_r2 ${params.ignore_r2}" : ""), - meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_r2 ${params.ignore_3prime_r2}": "") + meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_3prime_r2 ${params.ignore_3prime_r2}": "") ].join(' ').trim() } publishDir = [ [ @@ -229,11 +231,6 @@ process { mode: params.publish_dir_mode, pattern: "*bedGraph.gz" ], - [ - path: { "${params.outdir}/${params.aligner}/methylation_calls/stranded_CpG_report" }, - mode: params.publish_dir_mode, - pattern: "*CpG_report.txt.gz" - ], [ path: { "${params.outdir}/${params.aligner}/methylation_calls/splitting_report" }, mode: params.publish_dir_mode, @@ -250,11 +247,6 @@ process { withName: BISMARK_COVERAGE2CYTOSINE { ext.args = params.nomeseq ? '--nome-seq' : '' publishDir = [ - [ - path: { "${params.outdir}/bismark/coverage2cytosine/coverage" }, - mode: params.publish_dir_mode, - pattern: "*.cov.gz" - ], publishDir = [ path: { "${params.outdir}/bismark/coverage2cytosine/summaries" }, mode: params.publish_dir_mode, diff --git a/conf/test.config b/conf/test.config index 6843627b..ca5e25cb 100644 --- a/conf/test.config +++ b/conf/test.config @@ -17,9 +17,9 @@ params { max_time = 6.h // Input data - input = "$test_data_base/samplesheet/samplesheet_test.csv" + input = "${params.test_data_base}/samplesheet/samplesheet_test.csv" // Genome references - fasta = "$test_data_base/reference/genome.fa" - fasta_index = "$test_data_base/reference/genome.fa.fai" + fasta = "${params.test_data_base}/reference/genome.fa" + fasta_index = "${params.test_data_base}/reference/genome.fa.fai" } diff --git a/conf/test_full.config b/conf/test_full.config index 1f40c155..60dbdf96 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -10,12 +10,14 @@ ---------------------------------------------------------------------------------------- */ +cleanup = true + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' // Input data - input = "https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/samplesheet/samplesheet_full.csv" + input = "${params.test_data_base}/samplesheet/samplesheet_full.csv" genome = 'GRCh38' } diff --git a/docs/usage.md b/docs/usage.md index 2b446c25..899746f8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -128,7 +128,30 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` -## Updating the pipeline +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). +> The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run nf-core/methylseq -profile docker -params-file params.yaml +``` + +with `params.yaml` containing: + +```yaml +input: './samplesheet.csv' +outdir: './results/' +genome: 'GRCh37' +input: 'data' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + +### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -144,6 +167,10 @@ First, go to the [nf-core/methylseq releases page](https://github.com/nf-core/me This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +> 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. + ## Core Nextflow arguments > **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). @@ -152,7 +179,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -176,8 +203,10 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) - `charliecloud` - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) +- `apptainer` + - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. ### `-resume` @@ -195,29 +224,21 @@ Specify the path to a specific config file (this is a core Nextflow command). Se Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. -For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: +To change the resource requests, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. -```console -[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' +### Custom Containers + +In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. However in some cases the pipeline specified version maybe out of date. -Caused by: - Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) +To use a different container from the default container or conda environment specified in a pipeline, please see the [updating tool versions](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. -Command executed: - STAR \ - --genomeDir star \ - --readFilesIn WT_REP1_trimmed.fq.gz \ - --runThreadN 2 \ - --outFileNamePrefix WT_REP1. \ - +### Custom Tool Arguments -Command exit status: - 137 +A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. -Command output: - (empty) +To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. +```console Command error: .command.sh: line 9: 30 Killed STAR --genomeDir star --readFilesIn WT_REP1_trimmed.fq.gz --runThreadN 2 --outFileNamePrefix WT_REP1. Work dir: diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 33cd4f6e..9b34804d 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -2,6 +2,7 @@ // This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. // +import nextflow.Nextflow import org.everit.json.schema.Schema import org.everit.json.schema.loader.SchemaLoader import org.everit.json.schema.ValidationException @@ -83,6 +84,7 @@ class NfcoreSchema { 'stub-run', 'test', 'w', + 'with-apptainer', 'with-charliecloud', 'with-conda', 'with-dag', @@ -177,7 +179,7 @@ class NfcoreSchema { } if (has_error) { - System.exit(1) + Nextflow.error('Exiting!') } } diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 5c8bbac6..04cfeea5 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the main.nf workflow in the nf-core/methylseq pipeline // +import nextflow.Nextflow + class WorkflowMain { // @@ -20,7 +22,7 @@ class WorkflowMain { // // Generate help string // - public static String help(workflow, params, log) { + public static String help(workflow, params) { def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" def help_string = '' help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) @@ -33,7 +35,7 @@ class WorkflowMain { // // Generate parameter summary log string // - public static String paramsSummaryLog(workflow, params, log) { + public static String paramsSummaryLog(workflow, params) { def summary_log = '' summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) @@ -48,7 +50,7 @@ class WorkflowMain { public static void initialise(workflow, params, log) { // Print help to screen if required if (params.help) { - log.info help(workflow, params, log) + log.info help(workflow, params) System.exit(0) } @@ -60,7 +62,7 @@ class WorkflowMain { } // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params, log) + log.info paramsSummaryLog(workflow, params) // Validate workflow parameters via the JSON schema if (params.validate_params) { @@ -71,7 +73,7 @@ class WorkflowMain { NfcoreTemplate.checkConfigProvided(workflow, log) // Check that conda channels are set-up correctly - if (params.enable_conda) { + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { Utils.checkCondaChannels(log) } @@ -80,8 +82,7 @@ class WorkflowMain { // Check input has been provided if (!params.input) { - log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" - System.exit(1) + Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") } } // diff --git a/lib/WorkflowMethylseq.groovy b/lib/WorkflowMethylseq.groovy index 3e466067..f74108d9 100755 --- a/lib/WorkflowMethylseq.groovy +++ b/lib/WorkflowMethylseq.groovy @@ -2,6 +2,7 @@ // This file holds several functions specific to the workflow/methylseq.nf in the nf-core/methylseq pipeline // +import nextflow.Nextflow import groovy.text.SimpleTemplateEngine class WorkflowMethylseq { @@ -14,8 +15,9 @@ class WorkflowMethylseq { if (!params.fasta) { - log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - System.exit(1) + if (!params.bismark_index || params.aligner != 'bismark') { + Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." + } } } @@ -61,17 +63,19 @@ class WorkflowMethylseq { def description_html = engine.createTemplate(methods_text).make(meta) return description_html - }// + } + + // // Exit pipeline if incorrect --genome key provided // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } } diff --git a/main.nf b/main.nf index 9005c882..5dd70dfd 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,6 @@ nf-core/methylseq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/methylseq - Website: https://nf-co.re/methylseq Slack : https://nfcore.slack.com/channels/methylseq ---------------------------------------------------------------------------------------- diff --git a/modules.json b/modules.json index 634ae56d..2b51dee0 100644 --- a/modules.json +++ b/modules.json @@ -7,127 +7,127 @@ "nf-core": { "bismark/align": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/coverage2cytosine": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/deduplicate": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/genomepreparation": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/methylationextractor": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/report": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/summary": { "branch": "master", - "git_sha": "e84cc8f07607ad7970db9b6cf79b815b78d9cacd", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bwameth/align": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bwameth/index": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "cat/fastq": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", "installed_by": ["modules"] }, "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "8022c68e7403eecbd8ba9c49496f69f8c49d50f0", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "3dd73937b084b547f9272bc901e0f120a7913fd8", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "methyldackel/extract": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "methyldackel/mbias": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "picard/markduplicates": { "branch": "master", - "git_sha": "eca65aa4a5e2e192ac44d6962c8f9260f314ffb8", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "preseq/lcextrap": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "qualimap/bamqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "qualimap/bamqccram": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", "installed_by": ["modules"] }, "samtools/faidx": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/flagstat": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/index": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/sort": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/stats": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "trimgalore": { "branch": "master", - "git_sha": "b51a69e30973c71950225c817ad07a3337d22c40", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index 6983b1fa..3043b83b 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -2,10 +2,10 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'quay.io/biocontainers/python:3.8.3' }" + 'biocontainers/python:3.8.3' }" input: path samplesheet diff --git a/modules/nf-core/bismark/align/main.nf b/modules/nf-core/bismark/align/main.nf index eccb8550..d36f6726 100644 --- a/modules/nf-core/bismark/align/main.nf +++ b/modules/nf-core/bismark/align/main.nf @@ -2,10 +2,10 @@ process BISMARK_ALIGN { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bismark/coverage2cytosine/main.nf b/modules/nf-core/bismark/coverage2cytosine/main.nf index 5051f34c..f5429700 100644 --- a/modules/nf-core/bismark/coverage2cytosine/main.nf +++ b/modules/nf-core/bismark/coverage2cytosine/main.nf @@ -2,10 +2,10 @@ process BISMARK_COVERAGE2CYTOSINE { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.23.0" : null) + conda "bioconda::bismark=0.23.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(coverage_file) diff --git a/modules/nf-core/bismark/deduplicate/main.nf b/modules/nf-core/bismark/deduplicate/main.nf index be75b54d..7e238d77 100644 --- a/modules/nf-core/bismark/deduplicate/main.nf +++ b/modules/nf-core/bismark/deduplicate/main.nf @@ -2,10 +2,10 @@ process BISMARK_DEDUPLICATE { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/bismark/genomepreparation/main.nf b/modules/nf-core/bismark/genomepreparation/main.nf index 695a10f8..8f424443 100644 --- a/modules/nf-core/bismark/genomepreparation/main.nf +++ b/modules/nf-core/bismark/genomepreparation/main.nf @@ -2,10 +2,10 @@ process BISMARK_GENOMEPREPARATION { tag "$fasta" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: path fasta, stageAs: "BismarkIndex/*" diff --git a/modules/nf-core/bismark/methylationextractor/main.nf b/modules/nf-core/bismark/methylationextractor/main.nf index b8018ac7..2a3e5d22 100644 --- a/modules/nf-core/bismark/methylationextractor/main.nf +++ b/modules/nf-core/bismark/methylationextractor/main.nf @@ -2,10 +2,10 @@ process BISMARK_METHYLATIONEXTRACTOR { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/bismark/report/main.nf b/modules/nf-core/bismark/report/main.nf index 744c6cd1..804a141d 100644 --- a/modules/nf-core/bismark/report/main.nf +++ b/modules/nf-core/bismark/report/main.nf @@ -2,10 +2,10 @@ process BISMARK_REPORT { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(align_report), path(dedup_report), path(splitting_report), path(mbias) diff --git a/modules/nf-core/bismark/summary/main.nf b/modules/nf-core/bismark/summary/main.nf index 60854fba..399f960d 100644 --- a/modules/nf-core/bismark/summary/main.nf +++ b/modules/nf-core/bismark/summary/main.nf @@ -1,10 +1,10 @@ process BISMARK_SUMMARY { label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: val(bam) diff --git a/modules/nf-core/bwameth/align/main.nf b/modules/nf-core/bwameth/align/main.nf index 6d9c4431..47304035 100644 --- a/modules/nf-core/bwameth/align/main.nf +++ b/modules/nf-core/bwameth/align/main.nf @@ -2,10 +2,10 @@ process BWAMETH_ALIGN { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bwameth=0.2.2" : null) + conda "bioconda::bwameth=0.2.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bwameth:0.2.2--py_1' : - 'quay.io/biocontainers/bwameth:0.2.2--py_1' }" + 'biocontainers/bwameth:0.2.2--py_1' }" input: tuple val(meta), path(reads) @@ -24,7 +24,7 @@ process BWAMETH_ALIGN { def prefix = task.ext.prefix ?: "${meta.id}" def read_group = meta.read_group ? "-R ${meta.read_group}" : "" """ - INDEX=`find -L ${index} -name "*.bwameth.c2t" | sed 's/.bwameth.c2t//'` + INDEX=`find -L ${index} -name "*.bwameth.c2t" | sed 's/\\.bwameth.c2t\$//'` # Modify the timestamps so that bwameth doesn't complain about building the index # See https://github.com/nf-core/methylseq/pull/217 diff --git a/modules/nf-core/bwameth/index/main.nf b/modules/nf-core/bwameth/index/main.nf index 4c106826..1c789885 100644 --- a/modules/nf-core/bwameth/index/main.nf +++ b/modules/nf-core/bwameth/index/main.nf @@ -2,10 +2,10 @@ process BWAMETH_INDEX { tag "$fasta" label 'process_high' - conda (params.enable_conda ? "bioconda::bwameth=0.2.2" : null) + conda "bioconda::bwameth=0.2.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bwameth:0.2.2--py_1' : - 'quay.io/biocontainers/bwameth:0.2.2--py_1' }" + 'biocontainers/bwameth:0.2.2--py_1' }" input: path fasta, stageAs: "bwameth/*" diff --git a/modules/nf-core/cat/fastq/main.nf b/modules/nf-core/cat/fastq/main.nf index 4fa365d3..5021e6fc 100644 --- a/modules/nf-core/cat/fastq/main.nf +++ b/modules/nf-core/cat/fastq/main.nf @@ -2,10 +2,10 @@ process CAT_FASTQ { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "conda-forge::sed=4.7" : null) + conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'ubuntu:20.04' }" + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(reads, stageAs: "input*/*") diff --git a/modules/nf-core/cat/fastq/meta.yml b/modules/nf-core/cat/fastq/meta.yml index c836598e..8a39e309 100644 --- a/modules/nf-core/cat/fastq/meta.yml +++ b/modules/nf-core/cat/fastq/meta.yml @@ -1,6 +1,7 @@ name: cat_fastq description: Concatenates fastq files keywords: + - cat - fastq - concatenate tools: @@ -16,7 +17,7 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - reads: - type: list + type: file description: | List of input FastQ files to be concatenated. output: diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e05..ebc87273 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a0..c32657de 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,7 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom + - dump - version tools: - custom: diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 3bebb240..07d5e433 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -2,10 +2,10 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'quay.io/biocontainers/fastqc:0.11.9--0' }" + 'biocontainers/fastqc:0.11.9--0' }" input: tuple val(meta), path(reads) @@ -21,9 +21,15 @@ process FASTQC { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') """ - printf "%s\\n" $reads | while read f; do [[ \$f =~ ^${prefix}.* ]] || ln -s \$f ${prefix}_\$f ; done - fastqc $args --threads $task.cpus ${prefix}* + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/methyldackel/extract/main.nf b/modules/nf-core/methyldackel/extract/main.nf index 2ad1dc68..32951459 100644 --- a/modules/nf-core/methyldackel/extract/main.nf +++ b/modules/nf-core/methyldackel/extract/main.nf @@ -2,10 +2,10 @@ process METHYLDACKEL_EXTRACT { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? 'bioconda::methyldackel=0.6.0' : null) + conda "bioconda::methyldackel=0.6.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/methyldackel:0.6.0--h22771d5_0' : - 'quay.io/biocontainers/methyldackel:0.6.0--h22771d5_0' }" + 'biocontainers/methyldackel:0.6.0--h22771d5_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/methyldackel/extract/meta.yml b/modules/nf-core/methyldackel/extract/meta.yml index a831df16..9d777bb1 100644 --- a/modules/nf-core/methyldackel/extract/meta.yml +++ b/modules/nf-core/methyldackel/extract/meta.yml @@ -13,11 +13,10 @@ keywords: tools: - methyldackel: description: | - A (mostly) universal methylation extractor - for BS-seq experiments. - homepage: https://github.com/brentp/bwa-meth - documentation: https://github.com/brentp/bwa-meth - arxiv: arXiv:1401.1129 + Methylation caller from MethylDackel, a (mostly) universal methylation extractor + for methyl-seq experiments. + homepage: https://github.com/dpryan79/MethylDackel + documentation: https://github.com/dpryan79/MethylDackel/blob/master/README.md licence: ["MIT"] input: - meta: diff --git a/modules/nf-core/methyldackel/mbias/main.nf b/modules/nf-core/methyldackel/mbias/main.nf index 755e62f8..b30d3391 100644 --- a/modules/nf-core/methyldackel/mbias/main.nf +++ b/modules/nf-core/methyldackel/mbias/main.nf @@ -2,10 +2,10 @@ process METHYLDACKEL_MBIAS { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? 'bioconda::methyldackel=0.6.0' : null) + conda "bioconda::methyldackel=0.6.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/methyldackel:0.6.0--h22771d5_0' : - 'quay.io/biocontainers/methyldackel:0.6.0--h22771d5_0' }" + 'biocontainers/methyldackel:0.6.0--h22771d5_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/methyldackel/mbias/meta.yml b/modules/nf-core/methyldackel/mbias/meta.yml index 370f052a..75217232 100644 --- a/modules/nf-core/methyldackel/mbias/meta.yml +++ b/modules/nf-core/methyldackel/mbias/meta.yml @@ -14,11 +14,10 @@ keywords: tools: - methyldackel: description: | - A (mostly) universal methylation extractor - for BS-seq experiments. - homepage: https://github.com/brentp/bwa-meth - documentation: https://github.com/brentp/bwa-meth - arxiv: arXiv:1401.1129 + Read position methylation bias tools from MethylDackel, a (mostly) universal extractor + for methyl-seq experiments. + homepage: https://github.com/dpryan79/MethylDackel + documentation: https://github.com/dpryan79/MethylDackel/blob/master/README.md licence: ["MIT"] input: - meta: diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index a8159a57..1fc387be 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index ebc29b27..f93b5ee5 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: @@ -37,7 +38,7 @@ output: description: MultiQC report file pattern: "multiqc_report.html" - data: - type: dir + type: directory description: MultiQC data dir pattern: "multiqc_data" - plots: diff --git a/modules/nf-core/picard/markduplicates/main.nf b/modules/nf-core/picard/markduplicates/main.nf index d1f3aaa1..7ba8448f 100644 --- a/modules/nf-core/picard/markduplicates/main.nf +++ b/modules/nf-core/picard/markduplicates/main.nf @@ -2,10 +2,10 @@ process PICARD_MARKDUPLICATES { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::picard=2.27.4" : null) + conda "bioconda::picard=3.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/picard:2.27.4--hdfd78af_0' : - 'quay.io/biocontainers/picard:2.27.4--hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/picard:3.0.0--hdfd78af_1' : + 'biocontainers/picard:3.0.0--hdfd78af_1' }" input: tuple val(meta), path(bam) @@ -24,15 +24,15 @@ process PICARD_MARKDUPLICATES { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[Picard MarkDuplicates] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ picard \\ - -Xmx${avail_mem}g \\ + -Xmx${avail_mem}M \\ MarkDuplicates \\ $args \\ --INPUT $bam \\ diff --git a/modules/nf-core/preseq/lcextrap/main.nf b/modules/nf-core/preseq/lcextrap/main.nf index a98a922c..12546f0a 100644 --- a/modules/nf-core/preseq/lcextrap/main.nf +++ b/modules/nf-core/preseq/lcextrap/main.nf @@ -3,10 +3,10 @@ process PRESEQ_LCEXTRAP { label 'process_single' label 'error_ignore' - conda (params.enable_conda ? "bioconda::preseq=3.1.2" : null) + conda "bioconda::preseq=3.1.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/preseq:3.1.2--h445547b_2': - 'quay.io/biocontainers/preseq:3.1.2--h445547b_2' }" + 'biocontainers/preseq:3.1.2--h445547b_2' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/preseq/lcextrap/meta.yml b/modules/nf-core/preseq/lcextrap/meta.yml index f1be05a2..1391961c 100755 --- a/modules/nf-core/preseq/lcextrap/meta.yml +++ b/modules/nf-core/preseq/lcextrap/meta.yml @@ -10,7 +10,7 @@ tools: homepage: http://smithlabresearch.org/software/preseq/ documentation: http://smithlabresearch.org/wp-content/uploads/manual.pdf tool_dev_url: https://github.com/smithlabcode/preseq - doi: "" + licence: ["GPL"] input: diff --git a/modules/nf-core/qualimap/bamqc/main.nf b/modules/nf-core/qualimap/bamqc/main.nf index 3bfcb4c1..fef7307a 100644 --- a/modules/nf-core/qualimap/bamqc/main.nf +++ b/modules/nf-core/qualimap/bamqc/main.nf @@ -2,10 +2,10 @@ process QUALIMAP_BAMQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::qualimap=2.2.2d" : null) + conda "bioconda::qualimap=2.2.2d" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/qualimap:2.2.2d--1' : - 'quay.io/biocontainers/qualimap:2.2.2d--1' }" + 'biocontainers/qualimap:2.2.2d--1' }" input: tuple val(meta), path(bam) @@ -23,7 +23,7 @@ process QUALIMAP_BAMQC { prefix = task.ext.prefix ?: "${meta.id}" def collect_pairs = meta.single_end ? '' : '--collect-overlap-pairs' - def memory = task.memory.toGiga() + "G" + def memory = (task.memory.mega*0.8).intValue() + 'M' def regions = gff ? "--gff $gff" : '' def strandedness = 'non-strand-specific' @@ -34,7 +34,7 @@ process QUALIMAP_BAMQC { } """ unset DISPLAY - mkdir tmp + mkdir -p tmp export _JAVA_OPTIONS=-Djava.io.tmpdir=./tmp qualimap \\ --java-mem-size=$memory \\ diff --git a/modules/nf-core/qualimap/bamqccram/main.nf b/modules/nf-core/qualimap/bamqccram/main.nf index e136b8e2..327d8557 100644 --- a/modules/nf-core/qualimap/bamqccram/main.nf +++ b/modules/nf-core/qualimap/bamqccram/main.nf @@ -2,10 +2,10 @@ process QUALIMAP_BAMQCCRAM { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::qualimap=2.2.2d bioconda::samtools=1.15.1" : null) + conda "bioconda::qualimap=2.2.2d bioconda::samtools=1.16.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:61f6d4658ac88635fc37623af50bba77561988ab-0' : - 'quay.io/biocontainers/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:61f6d4658ac88635fc37623af50bba77561988ab-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:00d3c18496ddf07ea580fd00d1dd203cf31ab630-0' : + 'biocontainers/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:00d3c18496ddf07ea580fd00d1dd203cf31ab630-0' }" input: tuple val(meta), path(cram), path(crai) @@ -25,7 +25,7 @@ process QUALIMAP_BAMQCCRAM { prefix = task.ext.prefix ?: "${meta.id}" def collect_pairs = meta.single_end ? '' : '--collect-overlap-pairs' - def memory = task.memory.toGiga() + "G" + def memory = (task.memory.mega*0.8).intValue() + 'M' def regions = gff ? "--gff $gff" : '' def strandedness = 'non-strand-specific' @@ -36,7 +36,7 @@ process QUALIMAP_BAMQCCRAM { } """ unset DISPLAY - mkdir tmp + mkdir -p tmp export _JAVA_OPTIONS=-Djava.io.tmpdir=./tmp samtools view -hb -T ${fasta} ${cram} | diff --git a/modules/nf-core/qualimap/bamqccram/meta.yml b/modules/nf-core/qualimap/bamqccram/meta.yml index d72f203d..ea6c4d92 100644 --- a/modules/nf-core/qualimap/bamqccram/meta.yml +++ b/modules/nf-core/qualimap/bamqccram/meta.yml @@ -40,7 +40,7 @@ output: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - results: - type: dir + type: directory description: Qualimap results dir pattern: "*/*" - versions: diff --git a/modules/nf-core/samtools/faidx/main.nf b/modules/nf-core/samtools/faidx/main.nf index ef940db2..4dd0e5b0 100644 --- a/modules/nf-core/samtools/faidx/main.nf +++ b/modules/nf-core/samtools/faidx/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FAIDX { tag "$fasta" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf index c3152aca..eb7e72fc 100644 --- a/modules/nf-core/samtools/flagstat/main.nf +++ b/modules/nf-core/samtools/flagstat/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FLAGSTAT { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/samtools/flagstat/meta.yml b/modules/nf-core/samtools/flagstat/meta.yml index 95269063..954225df 100644 --- a/modules/nf-core/samtools/flagstat/meta.yml +++ b/modules/nf-core/samtools/flagstat/meta.yml @@ -14,7 +14,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf index e04e63e8..0b20aa4b 100644 --- a/modules/nf-core/samtools/index/main.nf +++ b/modules/nf-core/samtools/index/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_INDEX { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input) diff --git a/modules/nf-core/samtools/index/meta.yml b/modules/nf-core/samtools/index/meta.yml index e5cadbc2..8bd2fa6f 100644 --- a/modules/nf-core/samtools/index/meta.yml +++ b/modules/nf-core/samtools/index/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf index ab7f1cca..1e5181d4 100644 --- a/modules/nf-core/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_SORT { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam) @@ -21,9 +21,17 @@ process SAMTOOLS_SORT { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def sort_memory = (task.memory.mega/task.cpus).intValue() if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ - samtools sort $args -@ $task.cpus -o ${prefix}.bam -T $prefix $bam + samtools sort \\ + $args \\ + -@ $task.cpus \\ + -m ${sort_memory}M \\ + -o ${prefix}.bam \\ + -T $prefix \\ + $bam + cat <<-END_VERSIONS > versions.yml "${task.process}": samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') diff --git a/modules/nf-core/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml index 09289751..07328431 100644 --- a/modules/nf-core/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf index 9b0c3867..eb7f098b 100644 --- a/modules/nf-core/samtools/stats/main.nf +++ b/modules/nf-core/samtools/stats/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_STATS { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input), path(input_index) diff --git a/modules/nf-core/samtools/stats/meta.yml b/modules/nf-core/samtools/stats/meta.yml index cac50b1c..1d68a5d8 100644 --- a/modules/nf-core/samtools/stats/meta.yml +++ b/modules/nf-core/samtools/stats/meta.yml @@ -13,7 +13,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: @@ -23,13 +23,13 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - input: - type: file - description: BAM/CRAM file from alignment - pattern: "*.{bam,cram}" + type: file + description: BAM/CRAM file from alignment + pattern: "*.{bam,cram}" - input_index: - type: file - description: BAI/CRAI file from alignment - pattern: "*.{bai,crai}" + type: file + description: BAI/CRAI file from alignment + pattern: "*.{bai,crai}" - fasta: type: optional file description: Reference file the CRAM was created with diff --git a/modules/nf-core/trimgalore/main.nf b/modules/nf-core/trimgalore/main.nf index 5b45e4d7..dcb77ae7 100644 --- a/modules/nf-core/trimgalore/main.nf +++ b/modules/nf-core/trimgalore/main.nf @@ -2,22 +2,21 @@ process TRIMGALORE { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? 'bioconda::trim-galore=0.6.7' : null) + conda "bioconda::trim-galore=0.6.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/trim-galore:0.6.7--hdfd78af_0' : - 'quay.io/biocontainers/trim-galore:0.6.7--hdfd78af_0' }" + 'biocontainers/trim-galore:0.6.7--hdfd78af_0' }" input: tuple val(meta), path(reads) output: - tuple val(meta), path("*{trimmed,val}*.fq.gz"), emit: reads - tuple val(meta), path("*report.txt") , emit: log - path "versions.yml" , emit: versions - - tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true - tuple val(meta), path("*.html") , emit: html , optional: true - tuple val(meta), path("*.zip") , emit: zip , optional: true + tuple val(meta), path("*{3prime,5prime,trimmed,val}*.fq.gz"), emit: reads + tuple val(meta), path("*report.txt") , emit: log , optional: true + tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true + tuple val(meta), path("*.html") , emit: html , optional: true + tuple val(meta), path("*.zip") , emit: zip , optional: true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -38,10 +37,12 @@ process TRIMGALORE { // Added soft-links to original fastqs for consistent naming in MultiQC def prefix = task.ext.prefix ?: "${meta.id}" if (meta.single_end) { + def args_list = args.split("\\s(?=--)").toList() + args_list.removeAll { it.toLowerCase().contains('_r2 ') } """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz trim_galore \\ - $args \\ + ${args_list.join(' ')} \\ --cores $cores \\ --gzip \\ ${prefix}.fastq.gz diff --git a/modules/nf-core/trimgalore/meta.yml b/modules/nf-core/trimgalore/meta.yml index 439f566d..f84c4d77 100644 --- a/modules/nf-core/trimgalore/meta.yml +++ b/modules/nf-core/trimgalore/meta.yml @@ -36,7 +36,7 @@ output: description: | List of input adapter trimmed FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - pattern: "*.{fq.gz}" + pattern: "*{3prime,5prime,trimmed,val}*.fq.gz" - unpaired: type: file description: | diff --git a/nextflow.config b/nextflow.config index 01bcbe46..7e1f1b86 100644 --- a/nextflow.config +++ b/nextflow.config @@ -16,6 +16,8 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + // Qualimap options + bamqc_regions_file = null // MultiQC options multiqc_config = null multiqc_title = null @@ -63,7 +65,9 @@ params { // Bismark default is 0.2 (L,0,-0.2), Bowtie2 default is 0.6 (L,0,-0.6) meth_cutoff = null no_overlap = true + ignore_r1 = 0 ignore_r2 = 2 + ignore_3prime_r1 = 0 ignore_3prime_r2 = 2 known_splices = null local_alignment = false @@ -95,7 +99,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false // Config options @@ -136,18 +139,21 @@ try { profiles { - debug { process.beforeScript = 'echo $HOSTNAME' } + debug { + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false + } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false @@ -155,14 +161,17 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } docker { docker.enabled = true docker.userEmulation = true + conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' @@ -170,31 +179,48 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + conda.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } podman { podman.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } shifter { shifter.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false charliecloud.enabled = false + apptainer.enabled = false } charliecloud { charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false + } + apptainer { + apptainer.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false + charliecloud.enabled = false } gitpod { executor.name = 'local' @@ -230,6 +256,12 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set default registry for Docker and Podman independent of -profile +// Will not be used unless Docker / Podman are enabled +// Set to your registry if you have a mirror of containers +docker.registry = 'quay.io' +podman.registry = 'quay.io' + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true @@ -255,7 +287,7 @@ manifest { description = """Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '2.3.0' + version = '2.4.0' doi = '10.5281/zenodo.1343417' } diff --git a/nextflow_schema.json b/nextflow_schema.json index fe07cf05..ff4927ef 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -92,7 +92,7 @@ "mimetype": "text/plain", "pattern": "^\\S+\\.fn?a(sta)?$", "description": "Path to FASTA genome file", - "help_text": "If you have no genome reference available, the pipeline can build one using a FASTA file. This requires additional time and resources, so it's better to use a pre-build index if possible. You can use the command line option `--save_reference` to keep the generated references so that they can be added to your config and used again in the future." + "help_text": "If you have no genome reference available, the pipeline can build one using a FASTA file. This requires additional time and resources, so it's better to use a pre-build index if possible. You can use the command line option `--save_reference` to keep the generated references so that they can be added to your config and used again in the future. If aligner is Bismark and bismark_index is specified, this parameter is ignored." }, "fasta_index": { "type": "string", @@ -301,6 +301,13 @@ "help_text": "For paired-end reads it is theoretically possible that read_1 and read_2 overlap. To avoid scoring overlapping methylation calls twice, this is set to `true` by default. (Only methylation calls of read 1 are used since read 1 has historically higher quality basecalls than read 2). Whilst this option removes a bias towards more methylation calls in the center of sequenced fragments it may de facto remove a sizable proportion of the data. To count methylation data from both reads in overlapping regions, set this to `false`. ", "fa_icon": "fas fa-allergies" }, + "ignore_r1": { + "type": "integer", + "default": 0, + "description": "Ignore methylation in first n bases of 5' end of R1", + "help_text": "Ignore the first bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", + "fa_icon": "far fa-eye-slash" + }, "ignore_r2": { "type": "integer", "default": 2, @@ -308,6 +315,13 @@ "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, + "ignore_3prime_r1": { + "type": "integer", + "default": 0, + "description": "Ignore methylation in last n bases of 3' end of R1", + "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "fa_icon": "far fa-eye-slash" + }, "ignore_3prime_r2": { "type": "integer", "default": 2, @@ -374,6 +388,19 @@ }, "fa_icon": "far fa-circle" }, + "qualimap_options": { + "title": "Qualimap Options", + "type": "object", + "description": "Qualimap configurations", + "default": "", + "properties": { + "bamqc_regions_file": { + "type": "string", + "description": "A GFF or BED file containing the target regions which will be passed to Qualimap/Bamqc.", + "help_text": "Setting this option could be useful if you want calculate coverage stats over a list of regions, i.e. for targeted methylation sequencing data." + } + } + }, "skip_pipeline_steps": { "title": "Skip pipeline steps", "type": "object", @@ -500,11 +527,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile condo' instead of providing this parameter.", - "hidden": true } } }, @@ -556,7 +578,7 @@ "default": "master", "hidden": true, "fa_icon": "fas fa-users-cog", - "help_text": "Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`.\n\n```bash\n## Download and use config file with following git commit id\n--custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96\n```" + "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." }, "custom_config_base": { "type": "string", @@ -625,6 +647,9 @@ { "$ref": "#/definitions/bwa_meth_options" }, + { + "$ref": "#/definitions/qualimap_options" + }, { "$ref": "#/definitions/skip_pipeline_steps" }, diff --git a/nf-test.config b/nf-test.config index d8fd6cbe..cd432d43 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,7 +1,16 @@ config { - + // location for all nf-tests testsDir "tests" + + // nf-test directory including temporary files for each test workDir ".nf-test" - profile "test,docker" + // location of library folder that is added automatically to the classpath + libDir "tests/pipeline/lib/" + + // location of an optional nextflow.config file specific for executing tests + configFile "nextflow.config" + + // run all test with the defined docker profile from the main nextflow.config + profile "" } diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index faf87194..8fa2cc6e 100644 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -9,16 +9,15 @@ include { SAMTOOLS_FAIDX } from '../../modules/nf-core/samtools/fai workflow PREPARE_GENOME { main: - versions = Channel.empty() - - fasta = Channel.empty() - bismark_index = Channel.empty() - bwameth_index = Channel.empty() - fasta_index = Channel.empty() + ch_versions = Channel.empty() + ch_fasta = Channel.empty() + ch_bismark_index = Channel.empty() + ch_bwameth_index = Channel.empty() + ch_fasta_index = Channel.empty() // FASTA, if supplied if (params.fasta) { - fasta = file(params.fasta) + ch_fasta = Channel.value(file(params.fasta)) } // Aligner: bismark or bismark_hisat @@ -28,11 +27,11 @@ workflow PREPARE_GENOME { * Generate bismark index if not supplied */ if (params.bismark_index) { - bismark_index = file(params.bismark_index) + ch_bismark_index = Channel.value(file(params.bismark_index)) } else { - BISMARK_GENOMEPREPARATION(fasta) - bismark_index = BISMARK_GENOMEPREPARATION.out.index - versions = versions.mix(BISMARK_GENOMEPREPARATION.out.versions) + BISMARK_GENOMEPREPARATION(ch_fasta) + ch_bismark_index = BISMARK_GENOMEPREPARATION.out.index + ch_versions = ch_versions.mix(BISMARK_GENOMEPREPARATION.out.versions) } } @@ -43,30 +42,30 @@ workflow PREPARE_GENOME { * Generate bwameth index if not supplied */ if (params.bwa_meth_index) { - bwameth_index = file(params.bwa_meth_index) + ch_bwameth_index = Channel.value(file(params.bwa_meth_index)) } else { - BWAMETH_INDEX(fasta) - bwameth_index = BWAMETH_INDEX.out.index - versions = versions.mix(BWAMETH_INDEX.out.versions) + BWAMETH_INDEX(ch_fasta) + ch_bwameth_index = BWAMETH_INDEX.out.index + ch_versions = ch_versions.mix(BWAMETH_INDEX.out.versions) } /* * Generate fasta index if not supplied */ if (params.fasta_index) { - fasta_index = file(params.fasta_index) + ch_fasta_index = Channel.value(file(params.fasta_index)) } else { - SAMTOOLS_FAIDX([[:], fasta]) - fasta_index = SAMTOOLS_FAIDX.out.fai.map{ return(it[1])} - versions = versions.mix(SAMTOOLS_FAIDX.out.versions) + SAMTOOLS_FAIDX([[:], ch_fasta]) + ch_fasta_index = SAMTOOLS_FAIDX.out.fai.map{ return(it[1])} + ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) } } emit: - fasta - bismark_index - bwameth_index - fasta_index - versions + fasta = ch_fasta // channel: path(genome.fasta) + bismark_index = ch_bismark_index // channel: path(genome.fasta) + bwameth_index = ch_bwameth_index // channel: path(genome.fasta) + fasta_index = ch_fasta_index // channel: path(genome.fasta) + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] } diff --git a/tests/bismark/main.nf.test b/tests/bismark/main.nf.test deleted file mode 100644 index 51567718..00000000 --- a/tests/bismark/main.nf.test +++ /dev/null @@ -1,111 +0,0 @@ -import com.askimed.nf.test.util.FileUtil; - -nextflow_pipeline { - - name "bismark" - script "main.nf" - - test("Single End") { - when { - params { - aligner = "bismark" - save_reference = true - outdir = "$outputDir" - } - } - - then { - // Clean up anything from previous runs - FileUtil.deleteDirectory(new File(".nf-test/indexes/")); - new File(".nf-test/indexes/").mkdirs() - // Copy the ref genome to a stable location for next tests - FileUtil.copyDirectory( - new File("$outputDir/bismark/reference_genome/BismarkIndex/").getAbsolutePath(), - new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() - ) - - // Test results - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - assert new File("$outputDir/multiqc/bismark/multiqc_report.html").exists() - - assert new File("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt").exists() - assert new File("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt").exists() - assert path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt").md5 == "3038a90a6e513c3b1c0de540439eb101" - } - } - - test("Single End with index") { - when { - params { - aligner = "bismark" - // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" - cytosine_report = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS") { - when { - params { - aligner = "bismark" - skip_trimming = true - save_reference = true - rrbs = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS with index") { - when { - params { - aligner = "bismark" - skip_trimming = true - rrbs = true - save_reference = true - // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" - cytosine_report = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End NOMe-seq") { - when { - params { - aligner = "bismark" - nomeseq = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - } - } - -} diff --git a/tests/bismark/main.nf.test.snap b/tests/bismark/main.nf.test.snap deleted file mode 100644 index f7ba587c..00000000 --- a/tests/bismark/main.nf.test.snap +++ /dev/null @@ -1,32 +0,0 @@ -{ - "Single End RRBS with index": { - "content": [ - 32 - ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End NOMe-seq": { - "content": [ - 39 - ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End with index": { - "content": [ - 38 - ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End RRBS": { - "content": [ - 30 - ], - "timestamp": "2022-12-07T14:59:14+0000" - } -} \ No newline at end of file diff --git a/tests/bismark_hisat/main.nf.test b/tests/bismark_hisat/main.nf.test deleted file mode 100644 index 0daa6f7c..00000000 --- a/tests/bismark_hisat/main.nf.test +++ /dev/null @@ -1,89 +0,0 @@ -import com.askimed.nf.test.util.FileUtil; - -nextflow_pipeline { - - name "bismark_hisat" - script "main.nf" - - test("Single End") { - when { - params { - aligner = "bismark_hisat" - save_reference = true - outdir = "$outputDir" - } - } - - then { - // Clean up anything from previous runs - FileUtil.deleteDirectory(new File(".nf-test/indexes/")); - new File(".nf-test/indexes/").mkdirs() - // Copy the ref genome to a stable location for next tests - FileUtil.copyDirectory( - new File("$outputDir/bismark_hisat/reference_genome/BismarkIndex/").getAbsolutePath(), - new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() - ) - - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End with index") { - when { - params { - aligner = "bismark_hisat" - // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" - local_alignment = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS") { - when { - params { - aligner = "bismark_hisat" - save_reference = true - skip_trimming = true - rrbs = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS with index") { - when { - params { - aligner = "bismark_hisat" - skip_trimming = true - rrbs = true - // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" - local_alignment = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - -} diff --git a/tests/bismark_hisat/main.nf.test.snap b/tests/bismark_hisat/main.nf.test.snap deleted file mode 100644 index 38d4603c..00000000 --- a/tests/bismark_hisat/main.nf.test.snap +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Single End RRBS with index": { - "content": [ - 29 - ], - "timestamp": "2022-12-07T15:38:47+0000" - }, - "Single End with index": { - "content": [ - 35 - ], - "timestamp": "2022-12-07T15:38:47+0000" - }, - "Single End": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T15:38:47+0000" - }, - "Single End RRBS": { - "content": [ - 30 - ], - "timestamp": "2022-12-07T15:38:47+0000" - } -} \ No newline at end of file diff --git a/tests/bwameth/main.nf.test b/tests/bwameth/main.nf.test deleted file mode 100644 index 53b14dfa..00000000 --- a/tests/bwameth/main.nf.test +++ /dev/null @@ -1,93 +0,0 @@ -import com.askimed.nf.test.util.FileUtil; - -nextflow_pipeline { - - name "bwameth" - script "main.nf" - - test("Single End") { - when { - params { - aligner = "bwameth" - save_reference = true - outdir = "$outputDir" - } - } - - then { - // Clean up anything from previous runs - FileUtil.deleteDirectory(new File(".nf-test/indexes/")); - new File(".nf-test/indexes/").mkdirs() - // Copy the ref genome to a stable location for next tests - FileUtil.copyDirectory( - new File("$outputDir/bwameth/reference_genome/bwameth/").getAbsolutePath(), - new File(".nf-test/indexes/bwameth/").getAbsolutePath() - ) - - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - assert new File("$outputDir/multiqc/bwameth/multiqc_report.html").exists() - assert new File("$outputDir/bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam").exists() - assert new File("$outputDir/bwameth/reference_genome/bwameth/genome.fa.bwameth.c2t").exists() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End with index") { - when { - params { - aligner = "bwameth" - // Generated by previous test - bwa_meth_index = ".nf-test/indexes/bwameth/" - methyl_kit = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS") { - when { - params { - aligner = "bwameth" - save_reference = true - skip_trimming = true - rrbs = true - save_reference = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - - test("Single End RRBS with index") { - when { - params { - aligner = "bwameth" - skip_trimming = true - rrbs = true - // Generated by previous test - bwa_meth_index = ".nf-test/indexes/bwameth/" - methyl_kit = true - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - } - } - -} diff --git a/tests/bwameth/main.nf.test.snap b/tests/bwameth/main.nf.test.snap deleted file mode 100644 index b9ed3a35..00000000 --- a/tests/bwameth/main.nf.test.snap +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Single End RRBS with index": { - "content": [ - 34 - ], - "timestamp": "2022-12-07T15:05:21+0000" - }, - "Single End with index": { - "content": [ - 43 - ], - "timestamp": "2022-12-07T15:05:21+0000" - }, - "Single End": { - "content": [ - 44 - ], - "timestamp": "2022-12-07T15:05:21+0000" - }, - "Single End RRBS": { - "content": [ - 35 - ], - "timestamp": "2022-12-07T15:05:21+0000" - } -} \ No newline at end of file diff --git a/tests/clipping/main.nf.test b/tests/clipping/main.nf.test deleted file mode 100644 index 6e1cc9c9..00000000 --- a/tests/clipping/main.nf.test +++ /dev/null @@ -1,25 +0,0 @@ -nextflow_pipeline { - - name "Clipping" - script "main.nf" - - test("Should run with clipping params") { - when { - params { - aligner = "bismark" - em_seq = true - clip_r1 = 2 - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // TODO NFCORE_METHYLSEQ:METHYLSEQ:TRIMGALORE should include - // --three_prime_clip_r1 10 from the --em_seq preset - // --clip_r1 2 correctly overriding from --clip_r1 - } - } - -} diff --git a/tests/clipping/main.nf.test.snap b/tests/clipping/main.nf.test.snap deleted file mode 100644 index dde3180a..00000000 --- a/tests/clipping/main.nf.test.snap +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Should run with clipping params": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T15:00:07+0000" - } -} \ No newline at end of file diff --git a/tests/config/tags.yml b/tests/config/tags.yml new file mode 100644 index 00000000..db5bda67 --- /dev/null +++ b/tests/config/tags.yml @@ -0,0 +1,7 @@ +# tags for tests + +bismark: + +bismark_hisat: + +bwameth: diff --git a/tests/nextflow.config b/tests/nextflow.config deleted file mode 100644 index 8c256c7b..00000000 --- a/tests/nextflow.config +++ /dev/null @@ -1 +0,0 @@ -// This file is left intentionally blank diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test new file mode 100644 index 00000000..2bd75c9c --- /dev/null +++ b/tests/pipeline/bismark/main.nf.test @@ -0,0 +1,147 @@ +import com.askimed.nf.test.util.FileUtil; + +nextflow_pipeline { + + name "Test Workflow main.nf" + script "main.nf" + tag "bismark" + tag "pipeline" + + test("Bismark Single-End") { + when { + params { + aligner = "bismark" + save_reference = true + outdir = "$outputDir" + } + + } + + then { + // issue #278: Clean up anything from previous runs + FileUtil.deleteDirectory(new File(".nf-test/indexes/")); + new File(".nf-test/indexes/").mkdirs() + // Copy the ref genome to a stable location for next tests + FileUtil.copyDirectory( + new File("$outputDir/bismark/reference_genome/BismarkIndex/").getAbsolutePath(), + new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() + ) + + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Single-End With Index") { + when { + params { + aligner = "bismark" + // Generated by previous test + bismark_index = ".nf-test/indexes/BismarkIndex/" + cytosine_report = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + + test("Bismark Single-End With RRBS") { + when { + params { + aligner = "bismark" + skip_trimming = true + save_reference = true + rrbs = true + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_bismark_bt2_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_bt2_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Single-End With RRBS With Index") { + when { + params { + aligner = "bismark" + skip_trimming = true + rrbs = true + // Generated by previous test + bismark_index = ".nf-test/indexes/BismarkIndex/" + cytosine_report = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + + test("Bismark Single-End NOMe-seq") { + when { + params { + aligner = "bismark" + nomeseq = true + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_nomeseq") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_nomeseq_software_versions") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Clipping Params") { + when { + params { + aligner = "bismark" + em_seq = true + clip_r1 = 2 + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("clipping_params_software_versions") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) + } + } + +} diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap new file mode 100644 index 00000000..27cf8920 --- /dev/null +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -0,0 +1,59 @@ +{ + "clipping_params_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end_nomeseq_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end_nomeseq": { + "content": [ + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,5ce32b88b28766d6fa0bf91ea9df77d6", + "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", + "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", + "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", + "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", + "genome_results.txt:md5,29e911b75d7160e964530e9607110164" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end_rrbs_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end": { + "content": [ + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,5ce32b88b28766d6fa0bf91ea9df77d6", + "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", + "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", + "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", + "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", + "genome_results.txt:md5,29e911b75d7160e964530e9607110164" + ], + "timestamp": "2023-06-01T02:27:54+0000" + }, + "single_end_rrbs": { + "content": [ + "Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt:md5,fa089f6146df9667862970b6ddc998b0", + "SRR389222_sub1_bismark_bt2_splitting_report.txt:md5,f59eac7a30ec7f1f690aac921a5591c0", + "SRR389222_sub2.merged_bismark_bt2_splitting_report.txt:md5,43ca6c76e47ce2eb1c8dd6027475b5a6", + "genome_results.txt:md5,41becdcecee6ebbdd46f472629731892", + "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", + "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e" + ], + "timestamp": "2023-06-01T02:27:54+0000" + } +} \ No newline at end of file diff --git a/tests/pipeline/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test new file mode 100644 index 00000000..a368b722 --- /dev/null +++ b/tests/pipeline/bismark_hisat/main.nf.test @@ -0,0 +1,103 @@ +import com.askimed.nf.test.util.FileUtil; + +nextflow_pipeline { + + name "Test Workflow main.nf" + script "main.nf" + tag "bismark_hisat" + tag "pipeline" + + test("Bismark Hisat Single-End") { + when { + params { + aligner = "bismark_hisat" + save_reference = true + outdir = "$outputDir" + } + } + + then { + // Clean up anything from previous runs + FileUtil.deleteDirectory(new File(".nf-test/indexes/")); + new File(".nf-test/indexes/").mkdirs() + // Copy the ref genome to a stable location for next tests + FileUtil.copyDirectory( + new File("$outputDir/bismark_hisat/reference_genome/BismarkIndex/").getAbsolutePath(), + new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() + ) + + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_hisat2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") }, + { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Hisat Single-End With Index") { + when { + params { + aligner = "bismark_hisat" + // Generated by previous test + bismark_index = ".nf-test/indexes/BismarkIndex/" + local_alignment = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + + test("Bismark Hisat Single-End With RRBS") { + when { + params { + aligner = "bismark_hisat" + save_reference = true + skip_trimming = true + rrbs = true + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub1_bismark_hisat2_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") }, + { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Hisat Single-End With RRBS With Index") { + when { + params { + aligner = "bismark_hisat" + skip_trimming = true + rrbs = true + // Generated by previous test + bismark_index = ".nf-test/indexes/BismarkIndex/" + local_alignment = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + +} diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap new file mode 100644 index 00000000..c016095d --- /dev/null +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -0,0 +1,36 @@ +{ + "single_end_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:43:30+0000" + }, + "single_end_rrbs_software_versions": { + "content": [ + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-06-01T02:43:30+0000" + }, + "single_end": { + "content": [ + "Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt:md5,bdec8746a4f29c56ddbe37a92f7d35dd", + "SRR389222_sub1_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4020063dd8610b58b7b39c3ca62cef17", + "SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4db2155fdfae1cb9c7c758edba256e2e", + "genome_results.txt:md5,cd5d3b03821c038eb3bda24beb3ae938", + "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", + "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec" + ], + "timestamp": "2023-06-01T02:43:30+0000" + }, + "single_end_rrbs": { + "content": [ + "Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt:md5,76868c2015864a3f7c2b4cd1738e4765", + "SRR389222_sub1_bismark_hisat2_splitting_report.txt:md5,754ed7bf4d75acee466c48c05a11d331", + "SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt:md5,0629e1f8b4727efe8fd69500783eec61", + "genome_results.txt:md5,3afd1eb7d4dc72a05e4d4de887d791e3", + "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", + "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02" + ], + "timestamp": "2023-06-01T02:43:30+0000" + } +} \ No newline at end of file diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test new file mode 100644 index 00000000..95d783c0 --- /dev/null +++ b/tests/pipeline/bwameth/main.nf.test @@ -0,0 +1,108 @@ +import com.askimed.nf.test.util.FileUtil; + +nextflow_pipeline { + + name "Test Workflow main.nf" + script "main.nf" + tag "bwameth" + tag "pipeline" + + test("Bwameth Single-End") { + when { + params { + aligner = "bwameth" + save_reference = true + outdir = "$outputDir" + } + } + + then { + // Clean up anything from previous runs + FileUtil.deleteDirectory(new File(".nf-test/indexes/")); + new File(".nf-test/indexes/").mkdirs() + // Copy the ref genome to a stable location for next tests + FileUtil.copyDirectory( + new File("$outputDir/bwameth/reference_genome/bwameth/").getAbsolutePath(), + new File(".nf-test/indexes/bwameth/").getAbsolutePath() + ) + + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat"), + path("$outputDir/methyldackel/Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub1.markdup.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub2.markdup.sorted_CpG.bedGraph"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") } + ) + } + } + + test("Bwameth Single-End With Index") { + when { + params { + aligner = "bwameth" + // Generated by previous test + bwa_meth_index = ".nf-test/indexes/bwameth/" + methyl_kit = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + + test("Bwameth Single-End RRBS") { + when { + params { + aligner = "bwameth" + save_reference = true + skip_trimming = true + rrbs = true + save_reference = true + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat"), + path("$outputDir/methyldackel/Ecoli_10K_methylated.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub1.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub2.sorted_CpG.bedGraph"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") } + ) + } + } + + test("Bwameth Single End RRBS With Index") { + when { + params { + aligner = "bwameth" + skip_trimming = true + rrbs = true + // Generated by previous test + bwa_meth_index = ".nf-test/indexes/bwameth/" + methyl_kit = true + outdir = "$outputDir" + } + } + + then { + assert workflow.success + } + } + +} diff --git a/tests/pipeline/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap new file mode 100644 index 00000000..146f52e1 --- /dev/null +++ b/tests/pipeline/bwameth/main.nf.test.snap @@ -0,0 +1,42 @@ +{ + "single_end_software_versions": { + "content": [ + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-05-28T00:53:43+0000" + }, + "single_end_rrbs_software_versions": { + "content": [ + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" + ], + "timestamp": "2023-05-28T00:53:43+0000" + }, + "single_end": { + "content": [ + "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", + "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", + "SRR389222_sub2.flagstat:md5,2166a52f81f9b3881e8ac060876aa2d9", + "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", + "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", + "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", + "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", + "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", + "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0" + ], + "timestamp": "2023-05-28T00:53:43+0000" + }, + "single_end_rrbs": { + "content": [ + "Ecoli_10K_methylated.flagstat:md5,7c3bae0f9c700dde88785a7e94c6ae2c", + "SRR389222_sub1.flagstat:md5,897d500a710a56a7098172167fa71108", + "SRR389222_sub2.flagstat:md5,2d680c5657fc029b703b217be4af68e2", + "Ecoli_10K_methylated.sorted_CpG.bedGraph:md5,3656c54896013b76de978ab95c3f5975", + "SRR389222_sub1.sorted_CpG.bedGraph:md5,4475a9462ce86a019015b69b82a9d5a2", + "SRR389222_sub2.sorted_CpG.bedGraph:md5,18b8845d9d985d3c203ca6c9c40aab1b", + "genome_results.txt:md5,ca30867e16c5ca25c850c81d88563d41", + "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", + "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0" + ], + "timestamp": "2023-05-28T00:53:43+0000" + } +} \ No newline at end of file diff --git a/tests/pipeline/lib/UTILS.groovy b/tests/pipeline/lib/UTILS.groovy new file mode 100644 index 00000000..311403cc --- /dev/null +++ b/tests/pipeline/lib/UTILS.groovy @@ -0,0 +1,11 @@ +// Function to remove Nextflow version from software_versions.yml + +class UTILS { + public static String removeNextflowVersion(outputDir) { + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { + softwareVersions.Workflow.remove("Nextflow") + } + return softwareVersions + } +} diff --git a/tower.yml b/tower.yml index 2c5b52c9..f8332ee7 100644 --- a/tower.yml +++ b/tower.yml @@ -1,6 +1,8 @@ reports: multiqc_report.html: - display: "MultiQC report" + display: "MultiQC HTML report" + samplesheet.csv: + display: "Auto-created samplesheet with collated metadata and FASTQ paths" bismark_summary_report.html: display: "Bismark summary report" "bismark/reports/*report.html": diff --git a/workflows/methylseq.nf b/workflows/methylseq.nf index 21ad4d37..9ce22207 100644 --- a/workflows/methylseq.nf +++ b/workflows/methylseq.nf @@ -87,13 +87,13 @@ def multiqc_report = [] workflow METHYLSEQ { - versions = Channel.empty() + ch_versions = Channel.empty() // // SUBWORKFLOW: Prepare any required reference genome indices // PREPARE_GENOME() - versions = versions.mix(PREPARE_GENOME.out.versions) + ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) // // SUBWORKFLOW: Read in samplesheet, validate and stage input files @@ -117,7 +117,7 @@ workflow METHYLSEQ { return [ meta, fastq.flatten() ] } .set { ch_fastq } - versions = versions.mix(INPUT_CHECK.out.versions) + ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) // // MODULE: Concatenate FastQ files from same sample if required @@ -128,7 +128,7 @@ workflow METHYLSEQ { .reads .mix(ch_fastq.single) .set { ch_cat_fastq } - versions = versions.mix(CAT_FASTQ.out.versions.first()) + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first()) // // MODULE: Run FastQC @@ -136,7 +136,7 @@ workflow METHYLSEQ { FASTQC ( ch_cat_fastq ) - versions = versions.mix(FASTQC.out.versions.first()) + ch_versions = ch_versions.mix(FASTQC.out.versions.first()) /* * MODULE: Run TrimGalore! @@ -144,7 +144,7 @@ workflow METHYLSEQ { if (!params.skip_trimming) { TRIMGALORE(ch_cat_fastq) reads = TRIMGALORE.out.reads - versions = versions.mix(TRIMGALORE.out.versions.first()) + ch_versions = ch_versions.mix(TRIMGALORE.out.versions.first()) } else { reads = ch_cat_fastq } @@ -167,7 +167,7 @@ workflow METHYLSEQ { params.skip_deduplication || params.rrbs, params.cytosine_report || params.nomeseq ) - versions = versions.mix(BISMARK.out.versions.unique{ it.baseName }) + ch_versions = ch_versions.mix(BISMARK.out.versions.unique{ it.baseName }) ch_bam = BISMARK.out.bam ch_dedup = BISMARK.out.dedup ch_aligner_mqc = BISMARK.out.mqc @@ -182,7 +182,7 @@ workflow METHYLSEQ { PREPARE_GENOME.out.fasta_index, params.skip_deduplication || params.rrbs, ) - versions = versions.mix(BWAMETH.out.versions.unique{ it.baseName }) + ch_versions = ch_versions.mix(BWAMETH.out.versions.unique{ it.baseName }) ch_bam = BWAMETH.out.bam ch_dedup = BWAMETH.out.dedup ch_aligner_mqc = BWAMETH.out.mqc @@ -193,9 +193,9 @@ workflow METHYLSEQ { */ QUALIMAP_BAMQC ( ch_dedup, - [] + params.bamqc_regions_file ? Channel.fromPath( params.bamqc_regions_file, checkIfExists: true ).toList() : [] ) - versions = versions.mix(QUALIMAP_BAMQC.out.versions.first()) + ch_versions = ch_versions.mix(QUALIMAP_BAMQC.out.versions.first()) /* * MODULE: Run Preseq @@ -203,10 +203,10 @@ workflow METHYLSEQ { PRESEQ_LCEXTRAP ( ch_bam ) - versions = versions.mix(PRESEQ_LCEXTRAP.out.versions.first()) + ch_versions = ch_versions.mix(PRESEQ_LCEXTRAP.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - versions.unique().collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // @@ -238,7 +238,7 @@ workflow METHYLSEQ { ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() - versions = versions.mix(MULTIQC.out.versions) + ch_versions = ch_versions.mix(MULTIQC.out.versions) } }