From c19671dca974354978c9bc1711fca6fe681bdb0b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 8 Oct 2021 15:02:42 +0000 Subject: [PATCH] Subworkflow Infrastructure (#662) * feat(subworkflows): Add align_bowtie2 subworkflow For testing CI setup * test(align_bowtie2): Add initial list of changes to test * test(align_bowtie2): Add initial test * refactor: Use tags to run subworkflows ci For every underlying module used by workflow and allow the modules pytest-modules definition be the source of truth. * refactor: Use individual directories for subworkflows * docs(align_bowtie2): Add initial meta.yml Copied most of it from the bowtie2/align module. * fix(align_bowtie2): Fix module include paths * test(bam_sort_samtools): Add initial test * ci(bam_sort_samtools): Add modules that trigger the tag * test(bam_stats_samtools): Add initial test * ci(bam_stats_samtools): Add keys to pick up changes * docs(bam_samtools): Add initial meta.yml * test(align_bowtie2): Fix path to subworkflow * test(align_bowtie2): Update entry point * fix(bam_sort_samtools): Update include paths * test(bam_sort_samtools): Fix path * style: Clean up addParams * test(samtools_sort): Add suffix for test * test(align_bowtie2): Add samtools_options for suffix * test(bam_stats_samtools): Update path * test(bam_stats_samtools): Use stats input Otherwise it's just an example of how it's used in the bam_sort_samtools subworkflow * ci(linting): Skip module linting of subworkflows * ci(linting): Clean up startsWith statement * test(bam_stats_samtools): Use single end test data for single end test * test(bam_stats_samtools): Add expected files * test(align_bowtie2): Add paired-end test * test(align_bowtie2): Sort order of output * test(align_bowtie2): Update hashes * docs(align_bowtie2): Fix typo * test(align_bowtie2): Update samtools output names * test(align_bowtie2): Remove md5sums for bam/bai * feat(subworkflows): Add nextflow.configs These can be used for default settings in the future. They can then be included in the conf/modules.config so that the params don't have to be duplicated in the root nextflow.config. * docs(subworkflows): Include modules instead of tools * fix: Update to versions * chore(align_bowtie2): Remove duplicate tag * style: Format yamls * test(subworkflows): Only check versions for modules * chore: Update subworkflows to match rnaseq dev * fix(subworkflows): Update paths * fix(bam_sort_samtools): Fix sort parameters for testing * Apply suggestions from code review Co-authored-by: Harshil Patel * docs: Update TODOs with a message * ci: Try using a matrix for strategy * ci: Try passing an array * Revert "ci: Try passing an array" This reverts commit d3611fcd8332bbb9a8501e8dd299d0a623aaecaa. Co-authored-by: Harshil Patel --- .github/workflows/nf-core-linting.yml | 2 + .github/workflows/pytest-workflow.yml | 7 +- subworkflows/nf-core/align_bowtie2/main.nf | 47 +++++++++++ subworkflows/nf-core/align_bowtie2/meta.yml | 50 ++++++++++++ .../nf-core/align_bowtie2/nextflow.config | 2 + .../nf-core/bam_sort_samtools/main.nf | 53 ++++++++++++ .../nf-core/bam_sort_samtools/meta.yml | 41 ++++++++++ .../nf-core/bam_sort_samtools/nextflow.config | 1 + .../nf-core/bam_stats_samtools/main.nf | 33 ++++++++ .../nf-core/bam_stats_samtools/meta.yml | 43 ++++++++++ .../bam_stats_samtools/nextflow.config | 1 + tests/config/pytest_subworkflows.yml | 11 +++ tests/modules/samtools/sort/main.nf | 2 +- tests/modules/samtools/sort/test.yml | 4 +- .../nf-core/align_bowtie2/main.nf | 27 +++++++ .../nf-core/align_bowtie2/test.yml | 81 +++++++++++++++++++ .../nf-core/bam_sort_samtools/main.nf | 21 +++++ .../nf-core/bam_sort_samtools/test.yml | 47 +++++++++++ .../nf-core/bam_stats_samtools/main.nf | 23 ++++++ .../nf-core/bam_stats_samtools/test.yml | 31 +++++++ tests/test_versions_yml.py | 2 +- 21 files changed, 524 insertions(+), 5 deletions(-) create mode 100644 subworkflows/nf-core/align_bowtie2/main.nf create mode 100644 subworkflows/nf-core/align_bowtie2/meta.yml create mode 100644 subworkflows/nf-core/align_bowtie2/nextflow.config create mode 100644 subworkflows/nf-core/bam_sort_samtools/main.nf create mode 100644 subworkflows/nf-core/bam_sort_samtools/meta.yml create mode 100644 subworkflows/nf-core/bam_sort_samtools/nextflow.config create mode 100644 subworkflows/nf-core/bam_stats_samtools/main.nf create mode 100644 subworkflows/nf-core/bam_stats_samtools/meta.yml create mode 100644 subworkflows/nf-core/bam_stats_samtools/nextflow.config create mode 100644 tests/config/pytest_subworkflows.yml create mode 100644 tests/subworkflows/nf-core/align_bowtie2/main.nf create mode 100644 tests/subworkflows/nf-core/align_bowtie2/test.yml create mode 100644 tests/subworkflows/nf-core/bam_sort_samtools/main.nf create mode 100644 tests/subworkflows/nf-core/bam_sort_samtools/test.yml create mode 100644 tests/subworkflows/nf-core/bam_stats_samtools/main.nf create mode 100644 tests/subworkflows/nf-core/bam_stats_samtools/test.yml diff --git a/.github/workflows/nf-core-linting.yml b/.github/workflows/nf-core-linting.yml index 263b36b3..55b8c296 100644 --- a/.github/workflows/nf-core-linting.yml +++ b/.github/workflows/nf-core-linting.yml @@ -71,6 +71,8 @@ jobs: - name: Lint ${{ matrix.tags }} run: nf-core modules lint ${{ matrix.tags }} + # HACK + if: startsWith( matrix.tags, 'subworkflow' ) != true - uses: actions/cache@v2 with: diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 7cbb2689..6f395409 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -9,6 +9,11 @@ jobs: changes: name: Check for changes runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + filter: + ["tests/config/pytest_modules.yml", "tests/config/pytest_subworkflows.yml"] outputs: # Expose matched filters as job 'modules' output variable modules: ${{ steps.filter.outputs.changes }} @@ -18,7 +23,7 @@ jobs: - uses: dorny/paths-filter@v2 id: filter with: - filters: "tests/config/pytest_modules.yml" + filters: ${{ matrix.filter }} test: runs-on: ubuntu-20.04 diff --git a/subworkflows/nf-core/align_bowtie2/main.nf b/subworkflows/nf-core/align_bowtie2/main.nf new file mode 100644 index 00000000..ec453f8d --- /dev/null +++ b/subworkflows/nf-core/align_bowtie2/main.nf @@ -0,0 +1,47 @@ +// +// Alignment with Bowtie2 +// + +params.align_options = [:] +params.samtools_sort_options = [:] +params.samtools_index_options = [:] +params.samtools_stats_options = [:] + +include { BOWTIE2_ALIGN } from '../../../modules/bowtie2/align/main' addParams( options: params.align_options ) +include { BAM_SORT_SAMTOOLS } from '../bam_sort_samtools/main' addParams( sort_options: params.samtools_sort_options, index_options: params.samtools_index_options, stats_options: params.samtools_stats_options ) + +workflow ALIGN_BOWTIE2 { + take: + reads // channel: [ val(meta), [ reads ] ] + index // channel: /path/to/bowtie2/index/ + + main: + + ch_versions = Channel.empty() + + // + // Map reads with Bowtie2 + // + BOWTIE2_ALIGN ( reads, index ) + ch_versions = ch_versions.mix(BOWTIE2_ALIGN.out.versions.first()) + + // + // Sort, index BAM file and run samtools stats, flagstat and idxstats + // + BAM_SORT_SAMTOOLS ( BOWTIE2_ALIGN.out.bam ) + ch_versions = ch_versions.mix(BAM_SORT_SAMTOOLS.out.versions) + + emit: + bam_orig = BOWTIE2_ALIGN.out.bam // channel: [ val(meta), bam ] + log_out = BOWTIE2_ALIGN.out.log // channel: [ val(meta), log ] + fastq = BOWTIE2_ALIGN.out.fastq // channel: [ val(meta), fastq ] + + bam = BAM_SORT_SAMTOOLS.out.bam // channel: [ val(meta), [ bam ] ] + bai = BAM_SORT_SAMTOOLS.out.bai // channel: [ val(meta), [ bai ] ] + csi = BAM_SORT_SAMTOOLS.out.csi // channel: [ val(meta), [ csi ] ] + stats = BAM_SORT_SAMTOOLS.out.stats // channel: [ val(meta), [ stats ] ] + flagstat = BAM_SORT_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ] + idxstats = BAM_SORT_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ] + + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/align_bowtie2/meta.yml b/subworkflows/nf-core/align_bowtie2/meta.yml new file mode 100644 index 00000000..e149a212 --- /dev/null +++ b/subworkflows/nf-core/align_bowtie2/meta.yml @@ -0,0 +1,50 @@ +name: align_bowtie2 +description: Align reads to a reference genome using bowtie2 then sort with samtools +keywords: + - align + - fasta + - genome + - reference +modules: + - bowtie2/align + - samtools/sort + - samtools/index + - samtools/stats + - samtools/idxstats + - samtools/flagstat +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. + - index: + type: file + description: Bowtie2 genome index files + pattern: '*.ebwt' +# TODO Update when we decide on a standard for subworkflow docs +output: + - bam: + type: file + description: Output BAM file containing read alignments + pattern: '*.{bam}' + - versions: + type: file + description: File containing software versions + pattern: 'versions.yml' + - fastq: + type: file + description: Unaligned FastQ files + pattern: '*.fastq.gz' + - log: + type: file + description: Alignment log + pattern: '*.log' + # TODO Add samtools outputs +authors: + - '@drpatelh' diff --git a/subworkflows/nf-core/align_bowtie2/nextflow.config b/subworkflows/nf-core/align_bowtie2/nextflow.config new file mode 100644 index 00000000..89994865 --- /dev/null +++ b/subworkflows/nf-core/align_bowtie2/nextflow.config @@ -0,0 +1,2 @@ +params.align_options = [:] +params.samtools_options = [:] diff --git a/subworkflows/nf-core/bam_sort_samtools/main.nf b/subworkflows/nf-core/bam_sort_samtools/main.nf new file mode 100644 index 00000000..c9be6a56 --- /dev/null +++ b/subworkflows/nf-core/bam_sort_samtools/main.nf @@ -0,0 +1,53 @@ +// +// Sort, index BAM file and run samtools stats, flagstat and idxstats +// + +params.sort_options = [:] +params.index_options = [:] +params.stats_options = [:] + +include { SAMTOOLS_SORT } from '../../../modules/samtools/sort/main' addParams( options: params.sort_options ) +include { SAMTOOLS_INDEX } from '../../../modules/samtools/index/main' addParams( options: params.index_options ) +include { BAM_STATS_SAMTOOLS } from '../bam_stats_samtools/main' addParams( options: params.stats_options ) + +workflow BAM_SORT_SAMTOOLS { + take: + ch_bam // channel: [ val(meta), [ bam ] ] + + main: + + ch_versions = Channel.empty() + + SAMTOOLS_SORT ( ch_bam ) + ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions.first()) + + SAMTOOLS_INDEX ( SAMTOOLS_SORT.out.bam ) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) + + SAMTOOLS_SORT.out.bam + .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) + .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) + .map { + meta, bam, bai, csi -> + if (bai) { + [ meta, bam, bai ] + } else { + [ meta, bam, csi ] + } + } + .set { ch_bam_bai } + + BAM_STATS_SAMTOOLS ( ch_bam_bai ) + ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions) + + emit: + bam = SAMTOOLS_SORT.out.bam // channel: [ val(meta), [ bam ] ] + bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] + csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] + + stats = BAM_STATS_SAMTOOLS.out.stats // channel: [ val(meta), [ stats ] ] + flagstat = BAM_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ] + idxstats = BAM_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ] + + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/bam_sort_samtools/meta.yml b/subworkflows/nf-core/bam_sort_samtools/meta.yml new file mode 100644 index 00000000..a0e3f30b --- /dev/null +++ b/subworkflows/nf-core/bam_sort_samtools/meta.yml @@ -0,0 +1,41 @@ +name: bam_sort_samtools +description: Sort SAM/BAM/CRAM file +keywords: + - sort + - bam + - sam + - cram +modules: + - samtools/sort + - samtools/index + - samtools/stats + - samtools/idxstats + - samtools/flagstat +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: '*.{bam,cram,sam}' +# TODO Update when we decide on a standard for subworkflow docs +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: '*.{bam,cram,sam}' + - versions: + type: file + description: File containing software versions + pattern: 'versions.yml' +authors: + - '@drpatelh' + - '@ewels' diff --git a/subworkflows/nf-core/bam_sort_samtools/nextflow.config b/subworkflows/nf-core/bam_sort_samtools/nextflow.config new file mode 100644 index 00000000..2fd55747 --- /dev/null +++ b/subworkflows/nf-core/bam_sort_samtools/nextflow.config @@ -0,0 +1 @@ +params.options = [:] diff --git a/subworkflows/nf-core/bam_stats_samtools/main.nf b/subworkflows/nf-core/bam_stats_samtools/main.nf new file mode 100644 index 00000000..9276232c --- /dev/null +++ b/subworkflows/nf-core/bam_stats_samtools/main.nf @@ -0,0 +1,33 @@ +// +// Run SAMtools stats, flagstat and idxstats +// + +params.options = [:] + +include { SAMTOOLS_STATS } from '../../../modules/samtools/stats/main' addParams( options: params.options ) +include { SAMTOOLS_IDXSTATS } from '../../../modules/samtools/idxstats/main' addParams( options: params.options ) +include { SAMTOOLS_FLAGSTAT } from '../../../modules/samtools/flagstat/main' addParams( options: params.options ) + +workflow BAM_STATS_SAMTOOLS { + take: + ch_bam_bai // channel: [ val(meta), [ bam ], [bai/csi] ] + + main: + ch_versions = Channel.empty() + + SAMTOOLS_STATS ( ch_bam_bai ) + ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) + + SAMTOOLS_FLAGSTAT ( ch_bam_bai ) + ch_versions = ch_versions.mix(SAMTOOLS_FLAGSTAT.out.versions.first()) + + SAMTOOLS_IDXSTATS ( ch_bam_bai ) + ch_versions = ch_versions.mix(SAMTOOLS_IDXSTATS.out.versions.first()) + + emit: + stats = SAMTOOLS_STATS.out.stats // channel: [ val(meta), [ stats ] ] + flagstat = SAMTOOLS_FLAGSTAT.out.flagstat // channel: [ val(meta), [ flagstat ] ] + idxstats = SAMTOOLS_IDXSTATS.out.idxstats // channel: [ val(meta), [ idxstats ] ] + + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/bam_stats_samtools/meta.yml b/subworkflows/nf-core/bam_stats_samtools/meta.yml new file mode 100644 index 00000000..509c5c97 --- /dev/null +++ b/subworkflows/nf-core/bam_stats_samtools/meta.yml @@ -0,0 +1,43 @@ +name: samtools_stats +description: Produces comprehensive statistics from SAM/BAM/CRAM file +keywords: + - statistics + - counts + - bam + - sam + - cram +modules: + - samtools/stats + - samtools/idxstats + - samtools/flagstat +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: '*.{bam,cram,sam}' + - bai: + type: file + description: Index for BAM/CRAM/SAM file + pattern: '*.{bai,crai,sai}' +# TODO Update when we decide on a standard for subworkflow docs +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - stats: + type: file + description: File containing samtools stats output + pattern: '*.{stats}' + - versions: + type: file + description: File containing software versions + pattern: 'versions.yml' +authors: + - '@drpatelh' diff --git a/subworkflows/nf-core/bam_stats_samtools/nextflow.config b/subworkflows/nf-core/bam_stats_samtools/nextflow.config new file mode 100644 index 00000000..2fd55747 --- /dev/null +++ b/subworkflows/nf-core/bam_stats_samtools/nextflow.config @@ -0,0 +1 @@ +params.options = [:] diff --git a/tests/config/pytest_subworkflows.yml b/tests/config/pytest_subworkflows.yml new file mode 100644 index 00000000..a8ac84dc --- /dev/null +++ b/tests/config/pytest_subworkflows.yml @@ -0,0 +1,11 @@ +subworkflows/align_bowtie2: + - subworkflows/nf-core/align_bowtie2/** + - tests/subworkflows/nf-core/align_bowtie2/** + +subworkflows/bam_stats_samtools: + - subworkflows/nf-core/bam_stats_samtools/** + - tests/subworkflows/nf-core/bam_stats_samtools/** + +subworkflows/bam_sort_samtools: + - subworkflows/nf-core/bam_sort_samtools/** + - tests/subworkflows/nf-core/bam_sort_samtools/** diff --git a/tests/modules/samtools/sort/main.nf b/tests/modules/samtools/sort/main.nf index 91cd4d01..b76cdb1a 100644 --- a/tests/modules/samtools/sort/main.nf +++ b/tests/modules/samtools/sort/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { SAMTOOLS_SORT } from '../../../../modules/samtools/sort/main.nf' addParams( options: [:] ) +include { SAMTOOLS_SORT } from '../../../../modules/samtools/sort/main.nf' addParams( options: ['suffix': '.sorted'] ) workflow test_samtools_sort { input = [ [ id:'test', single_end:false ], // meta map diff --git a/tests/modules/samtools/sort/test.yml b/tests/modules/samtools/sort/test.yml index 477574fe..12e6669f 100644 --- a/tests/modules/samtools/sort/test.yml +++ b/tests/modules/samtools/sort/test.yml @@ -4,5 +4,5 @@ - samtools - samtools/sort files: - - path: output/samtools/test.bam - md5sum: bdc2d9e3f579f84df1e242207b627f89 + - path: output/samtools/test.sorted.bam + md5sum: bbb2db225f140e69a4ac577f74ccc90f diff --git a/tests/subworkflows/nf-core/align_bowtie2/main.nf b/tests/subworkflows/nf-core/align_bowtie2/main.nf new file mode 100644 index 00000000..9870242d --- /dev/null +++ b/tests/subworkflows/nf-core/align_bowtie2/main.nf @@ -0,0 +1,27 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BOWTIE2_BUILD } from '../../../../modules/bowtie2/build/main.nf' addParams( options: [:] ) +include { ALIGN_BOWTIE2 } from '../../../../subworkflows/nf-core/align_bowtie2/main.nf' addParams( 'samtools_sort_options': ['suffix': '.sorted'] ) + +workflow test_align_bowtie2_single_end { + input = [ [ id:'test', single_end:true ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] + ] + fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + + BOWTIE2_BUILD ( fasta ) + ALIGN_BOWTIE2 ( input, BOWTIE2_BUILD.out.index ) +} + +workflow test_align_bowtie2_paired_end { + input = [ [ id:'test', single_end:false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ] + ] + fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + + BOWTIE2_BUILD ( fasta ) + ALIGN_BOWTIE2 ( input, BOWTIE2_BUILD.out.index ) +} diff --git a/tests/subworkflows/nf-core/align_bowtie2/test.yml b/tests/subworkflows/nf-core/align_bowtie2/test.yml new file mode 100644 index 00000000..51261a14 --- /dev/null +++ b/tests/subworkflows/nf-core/align_bowtie2/test.yml @@ -0,0 +1,81 @@ +- name: align bowtie2 single-end + command: nextflow run ./tests/subworkflows/nf-core/align_bowtie2 -entry test_align_bowtie2_single_end -c tests/config/nextflow.config + tags: + - subworkflows/align_bowtie2 + - subworkflows/bam_sort_samtools + - subworkflows/bam_stats_samtools + # Modules + - bowtie2 + - bowtie2/align + - samtools + - samtools/index + - samtools/sort + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/index/bowtie2/genome.1.bt2 + md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf + - path: ./output/index/bowtie2/genome.2.bt2 + md5sum: 47b153cd1319abc88dda532462651fcf + - path: ./output/index/bowtie2/genome.3.bt2 + md5sum: 4ed93abba181d8dfab2e303e33114777 + - path: ./output/index/bowtie2/genome.4.bt2 + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/index/bowtie2/genome.rev.1.bt2 + md5sum: 52be6950579598a990570fbcf5372184 + - path: ./output/index/bowtie2/genome.rev.2.bt2 + md5sum: e3b4ef343dea4dd571642010a7d09597 + # samtools sort + - path: ./output/samtools/test.sorted.bam + - path: ./output/samtools/test.sorted.bam.bai + # samtools stats + - path: ./output/samtools/test.sorted.bam.flagstat + md5sum: e9ce9093133116bc54fd335cfe698372 + - path: ./output/samtools/test.sorted.bam.idxstats + md5sum: e16eb632f7f462514b0873c7ac8ac905 + - path: ./output/samtools/test.sorted.bam.stats + md5sum: 2d837cd72432cd856fca70d33f02ffb5 + +- name: align bowtie2 paired-end + command: nextflow run ./tests/subworkflows/nf-core/align_bowtie2 -entry test_align_bowtie2_paired_end -c tests/config/nextflow.config + tags: + - subworkflows/align_bowtie2 + - subworkflows/bam_sort_samtools + - subworkflows/bam_stats_samtools + # Modules + - bowtie2 + - bowtie2/align + - samtools + - samtools/index + - samtools/sort + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/index/bowtie2/genome.1.bt2 + md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf + - path: ./output/index/bowtie2/genome.2.bt2 + md5sum: 47b153cd1319abc88dda532462651fcf + - path: ./output/index/bowtie2/genome.3.bt2 + md5sum: 4ed93abba181d8dfab2e303e33114777 + - path: ./output/index/bowtie2/genome.4.bt2 + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/index/bowtie2/genome.rev.1.bt2 + md5sum: 52be6950579598a990570fbcf5372184 + - path: ./output/index/bowtie2/genome.rev.2.bt2 + md5sum: e3b4ef343dea4dd571642010a7d09597 + # samtools sort + - path: ./output/samtools/test.sorted.bam + - path: ./output/samtools/test.sorted.bam.bai + # samtools stats + - path: ./output/samtools/test.sorted.bam.flagstat + md5sum: 49f3d51a8804ce58fe9cecd2549d279b + - path: ./output/samtools/test.sorted.bam.idxstats + md5sum: 29ff2fa56d35b2a47625b8f517f1a947 + - path: ./output/samtools/test.sorted.bam.stats + md5sum: 98aa88a39d26244c89bd4e577953fb48 diff --git a/tests/subworkflows/nf-core/bam_sort_samtools/main.nf b/tests/subworkflows/nf-core/bam_sort_samtools/main.nf new file mode 100644 index 00000000..0dae6b2b --- /dev/null +++ b/tests/subworkflows/nf-core/bam_sort_samtools/main.nf @@ -0,0 +1,21 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BAM_SORT_SAMTOOLS } from '../../../../subworkflows/nf-core/bam_sort_samtools/main' addParams( sort_options: ['suffix': '.sorted'] ) + +workflow test_bam_sort_samtools_single_end { + input = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + ] + + BAM_SORT_SAMTOOLS ( input ) +} + +workflow test_bam_sort_samtools_paired_end { + input = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + + BAM_SORT_SAMTOOLS ( input ) +} diff --git a/tests/subworkflows/nf-core/bam_sort_samtools/test.yml b/tests/subworkflows/nf-core/bam_sort_samtools/test.yml new file mode 100644 index 00000000..e2fc27d8 --- /dev/null +++ b/tests/subworkflows/nf-core/bam_sort_samtools/test.yml @@ -0,0 +1,47 @@ +- name: bam sort samtools single-end + command: nextflow run ./tests/subworkflows/nf-core/bam_sort_samtools -entry test_bam_sort_samtools_single_end -c tests/config/nextflow.config + tags: + - subworkflows/bam_sort_samtools + - subworkflows/bam_stats_samtools + # Modules + - samtools + - samtools/index + - samtools/sort + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/samtools/test.sorted.bam + md5sum: e4c77897d6824ce4df486d1b100618af + - path: ./output/samtools/test.sorted.bam.bai + md5sum: a70940ce9ba2e700ec2984e0a6526099 + # samtools stats + - path: ./output/samtools/test.sorted.bam.flagstat + md5sum: 2191911d72575a2358b08b1df64ccb53 + - path: ./output/samtools/test.sorted.bam.idxstats + md5sum: 613e048487662c694aa4a2f73ca96a20 + - path: ./output/samtools/test.sorted.bam.stats + +- name: bam sort samtools paired-end + command: nextflow run ./tests/subworkflows/nf-core/bam_sort_samtools -entry test_bam_sort_samtools_paired_end -c tests/config/nextflow.config + tags: + - subworkflows/bam_sort_samtools + - subworkflows/bam_stats_samtools + # Modules + - samtools + - samtools/index + - samtools/sort + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/samtools/test.sorted.bam + md5sum: bbb2db225f140e69a4ac577f74ccc90f + - path: ./output/samtools/test.sorted.bam.bai + md5sum: 20c91e3a0fd4661d7cb967f40d2486ba + # samtools stats + - path: ./output/samtools/test.sorted.bam.flagstat + md5sum: 4f7ffd1e6a5e85524d443209ac97d783 + - path: ./output/samtools/test.sorted.bam.idxstats + md5sum: df60a8c8d6621100d05178c93fb053a2 + - path: ./output/samtools/test.sorted.bam.stats diff --git a/tests/subworkflows/nf-core/bam_stats_samtools/main.nf b/tests/subworkflows/nf-core/bam_stats_samtools/main.nf new file mode 100644 index 00000000..a390c3eb --- /dev/null +++ b/tests/subworkflows/nf-core/bam_stats_samtools/main.nf @@ -0,0 +1,23 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BAM_STATS_SAMTOOLS } from '../../../../subworkflows/nf-core/bam_stats_samtools/main' addParams( options: [:] ) + +workflow test_bam_stats_samtools_single_end { + input = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_sorted_bam'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_single_end_sorted_bam_bai'], checkIfExists: true) + ] + + BAM_STATS_SAMTOOLS ( input ) +} + +workflow test_bam_stats_samtools_paired_end { + input = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) + ] + + BAM_STATS_SAMTOOLS ( input ) +} diff --git a/tests/subworkflows/nf-core/bam_stats_samtools/test.yml b/tests/subworkflows/nf-core/bam_stats_samtools/test.yml new file mode 100644 index 00000000..d93c95a5 --- /dev/null +++ b/tests/subworkflows/nf-core/bam_stats_samtools/test.yml @@ -0,0 +1,31 @@ +- name: bam stats samtools single-end + command: nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools_single_end -c tests/config/nextflow.config + tags: + - subworkflows/bam_stats_samtools + # Modules + - samtools + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/samtools/test.single_end.sorted.bam.flagstat + md5sum: 2191911d72575a2358b08b1df64ccb53 + - path: ./output/samtools/test.single_end.sorted.bam.idxstats + md5sum: 613e048487662c694aa4a2f73ca96a20 + - path: ./output/samtools/test.single_end.sorted.bam.stats + +- name: bam stats samtools paired-end + command: nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools_paired_end -c tests/config/nextflow.config + tags: + - subworkflows/bam_stats_samtools + # Modules + - samtools + - samtools/stats + - samtools/idxstats + - samtools/flagstat + files: + - path: ./output/samtools/test.paired_end.sorted.bam.flagstat + md5sum: 4f7ffd1e6a5e85524d443209ac97d783 + - path: ./output/samtools/test.paired_end.sorted.bam.idxstats + md5sum: df60a8c8d6621100d05178c93fb053a2 + - path: ./output/samtools/test.paired_end.sorted.bam.stats diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index 759fc4d5..2f78ab2e 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -11,7 +11,7 @@ def _get_workflow_names(): To do so, recursively finds all test.yml files and parses their content. """ here = Path(__file__).parent.resolve() - pytest_workflow_files = here.glob("**/test.yml") + pytest_workflow_files = here.glob("modules/**/test.yml") for f in pytest_workflow_files: # test_config = yaml.safe_load(f.read_text()) test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader)