From d8028dc1c3ef64c2ee3494ce65d4f4a76c42bde9 Mon Sep 17 00:00:00 2001 From: Ramprasad Neethiraj <20065894+ramprasadn@users.noreply.github.com> Date: Thu, 3 Mar 2022 17:40:16 +0100 Subject: [PATCH 1/9] Add picard/sortvcf (#1370) * sortvcf * add files * update meta * update java mem * update documentation link * remove todo * review suggestions * fix test.yml * fix conda error * fix version code --- modules/picard/sortvcf/main.nf | 49 ++++++++++++++++++++ modules/picard/sortvcf/meta.yml | 40 ++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/picard/sortvcf/main.nf | 18 +++++++ tests/modules/picard/sortvcf/nextflow.config | 5 ++ tests/modules/picard/sortvcf/test.yml | 7 +++ 6 files changed, 123 insertions(+) create mode 100644 modules/picard/sortvcf/main.nf create mode 100644 modules/picard/sortvcf/meta.yml create mode 100644 tests/modules/picard/sortvcf/main.nf create mode 100644 tests/modules/picard/sortvcf/nextflow.config create mode 100644 tests/modules/picard/sortvcf/test.yml diff --git a/modules/picard/sortvcf/main.nf b/modules/picard/sortvcf/main.nf new file mode 100644 index 00000000..0f10c1ab --- /dev/null +++ b/modules/picard/sortvcf/main.nf @@ -0,0 +1,49 @@ +process PICARD_SORTVCF { + tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? "bioconda::picard=2.26.10" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/picard:2.26.10--hdfd78af_0' : + 'quay.io/biocontainers/picard:2.26.10--hdfd78af_0' }" + + input: + tuple val(meta), path(vcf) + path reference + path sequence_dict + + output: + tuple val(meta), path("*_sorted.vcf.gz"), emit: vcf + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def seq_dict = sequence_dict ? "-SEQUENCE_DICTIONARY $sequence_dict" : "" + def reference = reference ? "-REFERENCE_SEQUENCE $reference" : "" + def avail_mem = 3 + if (!task.memory) { + log.info '[Picard SortVcf] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' + } else { + avail_mem = task.memory.giga + } + + """ + picard \\ + SortVcf \\ + -Xmx${avail_mem}g \\ + --INPUT $vcf \\ + $args \\ + $seq_dict \\ + $reference \\ + --OUTPUT ${prefix}_sorted.vcf.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + picard: \$(picard SortVcf --version 2>&1 | grep -o 'Version:.*' | cut -f2- -d:) + END_VERSIONS + """ +} diff --git a/modules/picard/sortvcf/meta.yml b/modules/picard/sortvcf/meta.yml new file mode 100644 index 00000000..a2b46d5a --- /dev/null +++ b/modules/picard/sortvcf/meta.yml @@ -0,0 +1,40 @@ +name: picard_sortvcf +description: Sorts vcf files +keywords: + - sort + - vcf +tools: + - picard: + description: Java tools for working with NGS data in the BAM/CRAM/SAM and VCF format + homepage: https://broadinstitute.github.io/picard/ + documentation: https://broadinstitute.github.io/picard/command-line-overview.html#SortVcf + licence: ['MIT'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF file + pattern: "*.{vcf,vcf.gz}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - vcf: + type: file + description: Sorted VCF file + pattern: "*.{vcf}" + +authors: + - "@ramprasadn" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index de03a379..553128de 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1249,6 +1249,10 @@ picard/sortsam: - modules/picard/sortsam/** - tests/modules/picard/sortsam/** +picard/sortvcf: + - modules/picard/sortvcf/** + - tests/modules/picard/sortvcf/** + pirate: - modules/pirate/** - tests/modules/pirate/** diff --git a/tests/modules/picard/sortvcf/main.nf b/tests/modules/picard/sortvcf/main.nf new file mode 100644 index 00000000..a88c69fc --- /dev/null +++ b/tests/modules/picard/sortvcf/main.nf @@ -0,0 +1,18 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { PICARD_SORTVCF } from '../../../../modules/picard/sortvcf/main.nf' + +workflow test_picard_sortvcf { + + input = [ [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) + ] + + fasta = [ file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) ] + + dict = [ file(params.test_data['sarscov2']['genome']['genome_dict'], checkIfExists: true) ] + + PICARD_SORTVCF ( input, fasta, dict ) +} diff --git a/tests/modules/picard/sortvcf/nextflow.config b/tests/modules/picard/sortvcf/nextflow.config new file mode 100644 index 00000000..8730f1c4 --- /dev/null +++ b/tests/modules/picard/sortvcf/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} diff --git a/tests/modules/picard/sortvcf/test.yml b/tests/modules/picard/sortvcf/test.yml new file mode 100644 index 00000000..26271077 --- /dev/null +++ b/tests/modules/picard/sortvcf/test.yml @@ -0,0 +1,7 @@ +- name: picard sortvcf + command: nextflow run ./tests/modules/picard/sortvcf -entry test_picard_sortvcf -c ./tests/config/nextflow.config -c ./tests/modules/picard/sortvcf/nextflow.config + tags: + - picard + - picard/sortvcf + files: + - path: output/picard/test_sorted.vcf.gz From 72b96f4e504eef673f2b5c13560a9d90b669129b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Mar 2022 17:58:04 +0100 Subject: [PATCH 2/9] Add missing $prefix definition for MALT_RUN (#1373) * fix: remove left-over unnecessary code * Add forgotten prefix for the log * Update meta.yml * Update tests Co-authored-by: Sateesh Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/malt/run/main.nf | 3 ++- modules/malt/run/meta.yml | 2 +- tests/modules/malt/run/test.yml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/malt/run/main.nf b/modules/malt/run/main.nf index 61c02ec1..4e2e50c9 100644 --- a/modules/malt/run/main.nf +++ b/modules/malt/run/main.nf @@ -23,6 +23,7 @@ process MALT_RUN { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" def avail_mem = 6 if (!task.memory) { log.info '[MALT_RUN] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' @@ -39,7 +40,7 @@ process MALT_RUN { $args \\ --inFile ${fastqs.join(' ')} \\ -m $mode \\ - --index $index/ |&tee malt-run.log + --index $index/ |&tee ${prefix}-malt-run.log cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/malt/run/meta.yml b/modules/malt/run/meta.yml index ae4277a8..66f2d7a9 100644 --- a/modules/malt/run/meta.yml +++ b/modules/malt/run/meta.yml @@ -52,7 +52,7 @@ output: - log: type: file description: Log of verbose MALT stdout - pattern: "malt-run.log" + pattern: "*-malt-run.log" authors: - "@jfy133" diff --git a/tests/modules/malt/run/test.yml b/tests/modules/malt/run/test.yml index 335bc977..8ad44094 100644 --- a/tests/modules/malt/run/test.yml +++ b/tests/modules/malt/run/test.yml @@ -5,4 +5,4 @@ - malt/run files: - path: output/malt/test_1.rma6 - - path: output/malt/malt-run.log + - path: output/malt/test-malt-run.log From 251015c8bac16ecb55d738362503f17a84c45d18 Mon Sep 17 00:00:00 2001 From: Michael L Heuer Date: Fri, 4 Mar 2022 09:08:02 -0600 Subject: [PATCH 3/9] Add samtools index to yara_mapper module (#1353) * Add samtools index to yara_mapper module. * samtools sort required for index Co-authored-by: James A. Fellows Yates --- modules/yara/mapper/main.nf | 12 +++++++++--- modules/yara/mapper/meta.yml | 4 ++++ tests/modules/yara/mapper/test.yml | 3 +++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/yara/mapper/main.nf b/modules/yara/mapper/main.nf index 9c993ac7..15b39236 100644 --- a/modules/yara/mapper/main.nf +++ b/modules/yara/mapper/main.nf @@ -13,6 +13,7 @@ process YARA_MAPPER { output: tuple val(meta), path("*.mapped.bam"), emit: bam + tuple val(meta), path("*.mapped.bam.bai"), emit: bai path "versions.yml" , emit: versions when: @@ -28,7 +29,9 @@ process YARA_MAPPER { -t $task.cpus \\ -f bam \\ ${index}/yara \\ - $reads | samtools view -@ $task.cpus -hb -F4 > ${prefix}.mapped.bam + $reads | samtools view -@ $task.cpus -hb -F4 | samtools sort -@ $task.cpus > ${prefix}.mapped.bam + + samtools index -@ $task.cpus ${prefix}.mapped.bam cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -46,8 +49,11 @@ process YARA_MAPPER { ${reads[0]} \\ ${reads[1]} > output.bam - samtools view -@ $task.cpus -hF 4 -f 0x40 -b output.bam > ${prefix}_1.mapped.bam - samtools view -@ $task.cpus -hF 4 -f 0x80 -b output.bam > ${prefix}_2.mapped.bam + samtools view -@ $task.cpus -hF 4 -f 0x40 -b output.bam | samtools sort -@ $task.cpus > ${prefix}_1.mapped.bam + samtools view -@ $task.cpus -hF 4 -f 0x80 -b output.bam | samtools sort -@ $task.cpus > ${prefix}_2.mapped.bam + + samtools index -@ $task.cpus ${prefix}_1.mapped.bam + samtools index -@ $task.cpus ${prefix}_2.mapped.bam cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/yara/mapper/meta.yml b/modules/yara/mapper/meta.yml index 60089474..188e1d52 100644 --- a/modules/yara/mapper/meta.yml +++ b/modules/yara/mapper/meta.yml @@ -45,6 +45,10 @@ output: type: file description: Sorted BAM file pattern: "*.{bam}" + - bai: + type: file + description: Sorted BAM file index + pattern: "*.{bai}" authors: - "@apeltzer" diff --git a/tests/modules/yara/mapper/test.yml b/tests/modules/yara/mapper/test.yml index 186f70b4..3bfddc5b 100644 --- a/tests/modules/yara/mapper/test.yml +++ b/tests/modules/yara/mapper/test.yml @@ -5,6 +5,7 @@ - yara files: - path: output/yara/test.mapped.bam + - path: output/yara/test.mapped.bam.bai - path: output/yara/yara/yara.txt.size md5sum: 063987b3c3f747be7d2b8043c9d91000 - path: output/yara/yara/yara.lf.drs @@ -39,7 +40,9 @@ - yara files: - path: output/yara/test_2.mapped.bam + - path: output/yara/test_2.mapped.bam.bai - path: output/yara/test_1.mapped.bam + - path: output/yara/test_1.mapped.bam.bai - path: output/yara/yara/yara.txt.size md5sum: 063987b3c3f747be7d2b8043c9d91000 - path: output/yara/yara/yara.lf.drs From de0d57a5623ecb81d1bbc7ad73b5a8754b903d4c Mon Sep 17 00:00:00 2001 From: Benjamin Wingfield Date: Mon, 7 Mar 2022 18:02:40 +0000 Subject: [PATCH 4/9] implement plink2/score module (#1259) * implement plink2/score module * fix test yml * fix typo :( * set cpu * set mem * fix input process input block * fix tests Co-authored-by: Sateesh <33637490+sateeshperi@users.noreply.github.com> --- modules/plink2/score/main.nf | 39 +++++++++++++++ modules/plink2/score/meta.yml | 56 ++++++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/config/test_data.config | 3 +- tests/modules/plink2/score/main.nf | 24 ++++++++++ tests/modules/plink2/score/nextflow.config | 15 ++++++ tests/modules/plink2/score/test.yml | 16 +++++++ 7 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 modules/plink2/score/main.nf create mode 100644 modules/plink2/score/meta.yml create mode 100644 tests/modules/plink2/score/main.nf create mode 100644 tests/modules/plink2/score/nextflow.config create mode 100644 tests/modules/plink2/score/test.yml diff --git a/modules/plink2/score/main.nf b/modules/plink2/score/main.nf new file mode 100644 index 00000000..6f561322 --- /dev/null +++ b/modules/plink2/score/main.nf @@ -0,0 +1,39 @@ +process PLINK2_SCORE { + tag "$meta.id" + label 'process_low' + + conda (params.enable_conda ? "bioconda::plink2=2.00a2.3" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/plink2:2.00a2.3--h712d239_1' : + 'quay.io/biocontainers/plink2:2.00a2.3--h712d239_1' }" + + input: + tuple val(meta), path(pgen), path(psam), path(pvar) + path(scorefile) + + output: + tuple val(meta), path("*.sscore"), emit: score + path("versions.yml") , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def mem_mb = task.memory.toMega() // plink is greedy + """ + plink2 \\ + --threads $task.cpus \\ + --memory $mem_mb \\ + --pfile ${pgen.baseName} vzs \\ + --score ${scorefile} \\ + $args \\ + --out ${prefix} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + plink2: \$(plink2 --version 2>&1 | sed 's/^PLINK v//; s/ 64.*\$//' ) + END_VERSIONS + """ +} diff --git a/modules/plink2/score/meta.yml b/modules/plink2/score/meta.yml new file mode 100644 index 00000000..5dad6259 --- /dev/null +++ b/modules/plink2/score/meta.yml @@ -0,0 +1,56 @@ +name: plink2_score +description: Apply a scoring system to each sample in a plink 2 fileset +keywords: + - plink2 + - score +tools: + - plink2: + description: | + Whole genome association analysis toolset, designed to perform a range + of basic, large-scale analyses in a computationally efficient manner + homepage: http://www.cog-genomics.org/plink/2.0/ + documentation: http://www.cog-genomics.org/plink/2.0/general_usage + tool_dev_url: None + doi: "10.1186/s13742-015-0047-8" + licence: ['GPL v3'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - pgen: + type: file + description: PLINK 2 binary genotype table + pattern: "*.{pgen}" + - psam: + type: file + description: PLINK 2 sample information file + pattern: "*.{psam}" + - pvar: + type: file + description: PLINK 2 variant information file + pattern: "*.{pvar}" + - scorefile: + type: file + description: A text file containing variant identifiers and weights + pattern: "*.{scores,txt,scorefile}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - score: + type: file + description: A text file containing sample scores, in plink 2 .sscore format + pattern: "*.{sscore}" + +authors: + - "@nebfield" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 553128de..d6575ff1 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1273,6 +1273,10 @@ plink2/extract: - modules/plink2/extract/** - tests/modules/plink2/extract/** +plink2/score: + - modules/plink2/score/** + - tests/modules/plink2/score/** + plink2/vcf: - modules/plink2/vcf/** - tests/modules/plink2/vcf/** diff --git a/tests/config/test_data.config b/tests/config/test_data.config index dda10192..ce4f7ae8 100644 --- a/tests/config/test_data.config +++ b/tests/config/test_data.config @@ -119,7 +119,7 @@ params { genome_bed_gz_tbi = "${test_data_dir}/genomics/homo_sapiens/genome/genome.bed.gz.tbi" transcriptome_fasta = "${test_data_dir}/genomics/homo_sapiens/genome/transcriptome.fasta" genome2_fasta = "${test_data_dir}/genomics/homo_sapiens/genome/genome2.fasta" - genome_chain_gz = "${test_data_dir}/genomics/homo_sapiens/genome/genome.chain.gz" + genome_chain_gz = "${test_data_dir}/genomics/homo_sapiens/genome/genome.chain.gz" genome_21_fasta = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta" genome_21_fasta_fai = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta.fai" genome_21_dict = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/genome.dict" @@ -138,6 +138,7 @@ params { mills_and_1000g_indels_vcf_gz_tbi = "${test_data_dir}/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz.tbi" syntheticvcf_short_vcf_gz = "${test_data_dir}/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz" syntheticvcf_short_vcf_gz_tbi = "${test_data_dir}/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz.tbi" + syntheticvcf_short_score = "${test_data_dir}/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.score" gnomad_r2_1_1_sv_vcf_gz = "${test_data_dir}/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1-sv.vcf.gz" hapmap_3_3_hg38_21_vcf_gz = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/germlineresources/hapmap_3.3.hg38.vcf.gz" diff --git a/tests/modules/plink2/score/main.nf b/tests/modules/plink2/score/main.nf new file mode 100644 index 00000000..6a09e829 --- /dev/null +++ b/tests/modules/plink2/score/main.nf @@ -0,0 +1,24 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { PLINK2_VCF } from '../../../../modules/plink2/vcf/main.nf' +include { PLINK2_SCORE } from '../../../../modules/plink2/score/main.nf' + +workflow test_plink2_score { + input = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['genome']['syntheticvcf_short_vcf_gz'], checkIfExists: true) + ] + PLINK2_VCF ( input ) + + scorefile = file(params.test_data['homo_sapiens']['genome']['syntheticvcf_short_score'], checkIfExists: true) + + PLINK2_VCF.out.pgen + .concat(PLINK2_VCF.out.psam, PLINK2_VCF.out.pvar) + .groupTuple() + .map { it.flatten() } + .set { ch_target_genome } + + PLINK2_SCORE ( ch_target_genome, scorefile ) +} diff --git a/tests/modules/plink2/score/nextflow.config b/tests/modules/plink2/score/nextflow.config new file mode 100644 index 00000000..083e4666 --- /dev/null +++ b/tests/modules/plink2/score/nextflow.config @@ -0,0 +1,15 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + + // relabel input variants to a common scheme chr:pos:alt:ref + withName: PLINK2_VCF { + ext.args = '--set-missing-var-ids @:#:\\$1:\\$2' + } + + // scoring really needs an adjustment for small test dataset (n > 50 + // normally) + withName: PLINK2_SCORE { + ext.args = 'no-mean-imputation' + } +} diff --git a/tests/modules/plink2/score/test.yml b/tests/modules/plink2/score/test.yml new file mode 100644 index 00000000..7993cb34 --- /dev/null +++ b/tests/modules/plink2/score/test.yml @@ -0,0 +1,16 @@ +- name: plink2 score test_plink2_score + command: nextflow run tests/modules/plink2/score -entry test_plink2_score -c tests/config/nextflow.config + tags: + - plink2 + - plink2/score + files: + - path: output/plink2/test.pgen + md5sum: fac12ca9041d6950f6b7d60ac2120721 + - path: output/plink2/test.psam + md5sum: e6c714488754cb8448c3dfda08c4c0ea + - path: output/plink2/test.pvar.zst + md5sum: 98d59e9779a8b62d5032cd98b642a63b + - path: output/plink2/test.sscore + md5sum: 97bde840f69febd65f2c00e9243126e9 + - path: output/plink2/versions.yml + md5sum: 71499ab14e1583c88ced3a7a4f05bfa7 From b78a4a456762a4c59fd5023e70f36a27f76d4a97 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 8 Mar 2022 15:56:23 +0100 Subject: [PATCH 5/9] Fix for Maxbin2 emitting input files (#1376) * fix: remove left-over unnecessary code * Fix accidently emitting input * Fix tests --- modules/maxbin2/main.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/maxbin2/main.nf b/modules/maxbin2/main.nf index 7b818d00..a48df43f 100644 --- a/modules/maxbin2/main.nf +++ b/modules/maxbin2/main.nf @@ -29,8 +29,9 @@ process MAXBIN2 { def prefix = task.ext.prefix ?: "${meta.id}" def associate_files = reads ? "-reads $reads" : "-abund $abund" """ + mkdir input/ && mv $contigs input/ run_MaxBin.pl \\ - -contig $contigs \\ + -contig input/$contigs \\ $associate_files \\ -thread $task.cpus \\ $args \\ From e79bcd7d4e517b72045924c16bb778a2f074cf88 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 9 Mar 2022 14:55:31 +0100 Subject: [PATCH 6/9] Add git aware option to pytest commands (#1379) --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- .github/workflows/pytest-workflow.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b9f7a4e8..cfe07f88 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -27,6 +27,6 @@ Closes #XXX - [ ] Add a resource `label` - [ ] Use BioConda and BioContainers if possible to fulfil software requirements. - Ensure that the test works with either Docker / Singularity. Conda CI tests can be quite flaky: - - [ ] `PROFILE=docker pytest --tag --symlink --keep-workflow-wd` - - [ ] `PROFILE=singularity pytest --tag --symlink --keep-workflow-wd` - - [ ] `PROFILE=conda pytest --tag --symlink --keep-workflow-wd` + - [ ] `PROFILE=docker pytest --tag --symlink --keep-workflow-wd --git-aware` + - [ ] `PROFILE=singularity pytest --tag --symlink --keep-workflow-wd --git-aware` + - [ ] `PROFILE=conda pytest --tag --symlink --keep-workflow-wd --git-aware` diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index ee922c45..b2be6aa3 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -86,7 +86,7 @@ jobs: # Test the module - name: Run pytest-workflow # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware - name: Output log on failure if: failure() From b82d7abe7089a4b411e326c9e129faf03ba45741 Mon Sep 17 00:00:00 2001 From: "Thomas A. Christensen II" <25492070+MillironX@users.noreply.github.com> Date: Wed, 9 Mar 2022 10:56:35 -0600 Subject: [PATCH 7/9] Decrease indent in seqtk/seq versions.yml output (#1384) Signed-off-by: Thomas A. Christensen II <25492070+MillironX@users.noreply.github.com> Co-authored-by: Sateesh Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/seqtk/seq/main.nf | 4 ++-- tests/modules/seqtk/seq/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/seqtk/seq/main.nf b/modules/seqtk/seq/main.nf index 1fb03003..d1944ef9 100644 --- a/modules/seqtk/seq/main.nf +++ b/modules/seqtk/seq/main.nf @@ -33,8 +33,8 @@ process SEQTK_SEQ { gzip -c > ${prefix}.seqtk-seq.${extension}.gz cat <<-END_VERSIONS > versions.yml - "${task.process}": - seqtk: \$(echo \$(seqtk 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + "${task.process}": + seqtk: \$(echo \$(seqtk 2>&1) | sed 's/^.*Version: //; s/ .*\$//') END_VERSIONS """ } diff --git a/tests/modules/seqtk/seq/test.yml b/tests/modules/seqtk/seq/test.yml index 3e4532c6..c6937364 100644 --- a/tests/modules/seqtk/seq/test.yml +++ b/tests/modules/seqtk/seq/test.yml @@ -7,7 +7,7 @@ - path: output/seqtk/test.seqtk-seq.fasta.gz md5sum: 50d73992c8c7e56dc095ef47ec52a754 - path: output/seqtk/versions.yml - md5sum: 2b89cd4a6e28f35fcfbbd2188384f944 + md5sum: 6555e1061080c44f828de0b40b299e41 - name: seqtk seq test_seqtk_seq_fq command: nextflow run tests/modules/seqtk/seq -entry test_seqtk_seq_fq -c tests/config/nextflow.config @@ -18,4 +18,4 @@ - path: output/seqtk/test.seqtk-seq.fasta.gz md5sum: 2f009f1647971a97b4edec726a99dc1a - path: output/seqtk/versions.yml - md5sum: 3467a76d3540bee8f58de050512bddaa + md5sum: feb70feb3165d5c19fa50c16e46e6772 From 24f0bdd14ec32e0114aa6ee5337ddbd490ffd570 Mon Sep 17 00:00:00 2001 From: Michael J Cipriano <42848032+mjcipriano@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:36:05 -0500 Subject: [PATCH 8/9] added module seqkit replace (#1382) * added module seqkit replace * added when * removed extra line * Update modules/seqkit/replace/main.nf Co-authored-by: Robert A. Petit III * Updated meta * updated indents Co-authored-by: Cipriano Co-authored-by: Sateesh Peri <33637490+sateeshperi@users.noreply.github.com> Co-authored-by: Robert A. Petit III --- modules/seqkit/replace/main.nf | 41 ++++++++++++++++++++ modules/seqkit/replace/meta.yml | 41 ++++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/seqkit/replace/main.nf | 24 ++++++++++++ tests/modules/seqkit/replace/nextflow.config | 14 +++++++ tests/modules/seqkit/replace/test.yml | 21 ++++++++++ 6 files changed, 145 insertions(+) create mode 100644 modules/seqkit/replace/main.nf create mode 100644 modules/seqkit/replace/meta.yml create mode 100644 tests/modules/seqkit/replace/main.nf create mode 100644 tests/modules/seqkit/replace/nextflow.config create mode 100644 tests/modules/seqkit/replace/test.yml diff --git a/modules/seqkit/replace/main.nf b/modules/seqkit/replace/main.nf new file mode 100644 index 00000000..db189ef6 --- /dev/null +++ b/modules/seqkit/replace/main.nf @@ -0,0 +1,41 @@ +process SEQKIT_REPLACE { + tag "$meta.id" + label 'process_low' + + conda (params.enable_conda ? "bioconda::seqkit=2.1.0" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/seqkit:2.1.0--h9ee0642_0': + 'quay.io/biocontainers/seqkit:2.1.0--h9ee0642_0' }" + + input: + tuple val(meta), path(fastx) + + output: + tuple val(meta), path("*.fast*"), emit: fastx + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def extension = "fastq" + if ("$fastx" ==~ /.+\.fasta|.+\.fasta.gz|.+\.fa|.+\.fa.gz|.+\.fas|.+\.fas.gz|.+\.fna|.+\.fna.gz/) { + extension = "fasta" + } + def endswith = task.ext.suffix ?: "${extension}.gz" + """ + seqkit \\ + replace \\ + ${args} \\ + --threads ${task.cpus} \\ + -i ${fastx} \\ + -o ${prefix}.${endswith} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + seqkit: \$( seqkit | sed '3!d; s/Version: //' ) + END_VERSIONS + """ +} diff --git a/modules/seqkit/replace/meta.yml b/modules/seqkit/replace/meta.yml new file mode 100644 index 00000000..c15d04cb --- /dev/null +++ b/modules/seqkit/replace/meta.yml @@ -0,0 +1,41 @@ +name: seqkit_replace +description: Use seqkit to find/replace strings within sequences and sequence headers +keywords: + - seqkit + - replace +tools: + - seqkit: + description: Cross-platform and ultrafast toolkit for FASTA/Q file manipulation, written by Wei Shen. + homepage: https://bioinf.shenwei.me/seqkit/usage/ + documentation: https://bioinf.shenwei.me/seqkit/usage/ + tool_dev_url: https://github.com/shenwei356/seqkit/ + doi: "10.1371/journal.pone.016396" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fastx: + type: file + description: fasta/q file + pattern: "*.{fasta,fastq,fa,fq,fas,fna,faa}*" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - fastx: + type: file + description: fasta/q file with replaced values + pattern: "*.{fasta,fastq,fa,fq,fas,fna,faa}*" + +authors: + - "@mjcipriano" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index d6575ff1..a370f371 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1481,6 +1481,10 @@ seqkit/pair: - modules/seqkit/pair/** - tests/modules/seqkit/pair/** +seqkit/replace: + - modules/seqkit/replace/** + - tests/modules/seqkit/replace/** + seqkit/split2: - modules/seqkit/split2/** - tests/modules/seqkit/split2/** diff --git a/tests/modules/seqkit/replace/main.nf b/tests/modules/seqkit/replace/main.nf new file mode 100644 index 00000000..5c4058e7 --- /dev/null +++ b/tests/modules/seqkit/replace/main.nf @@ -0,0 +1,24 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { SEQKIT_REPLACE } from '../../../../modules/seqkit/replace/main.nf' +include { SEQKIT_REPLACE as SEQKIT_REPLACEUNCOMP } from '../../../../modules/seqkit/replace/main.nf' + +workflow test_seqkit_replace { + + input = [ [ id:'test' ], // meta map + [ file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) ] + ] + + SEQKIT_REPLACE ( input ) +} + +workflow test_seqkit_replace_uncomp { + + input = [ [ id:'test' ], // meta map + [ file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) ] + ] + + SEQKIT_REPLACEUNCOMP ( input ) +} diff --git a/tests/modules/seqkit/replace/nextflow.config b/tests/modules/seqkit/replace/nextflow.config new file mode 100644 index 00000000..8cec8505 --- /dev/null +++ b/tests/modules/seqkit/replace/nextflow.config @@ -0,0 +1,14 @@ +process { + + withName: 'SEQKIT_REPLACE' { + ext.args = "-s -p 'A' -r 'N'" + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + } + + withName: 'SEQKIT_REPLACEUNCOMP' { + ext.args = "-s -p 'T' -r 'N'" + ext.suffix = ".fasta" + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + } + +} diff --git a/tests/modules/seqkit/replace/test.yml b/tests/modules/seqkit/replace/test.yml new file mode 100644 index 00000000..94c3a5ef --- /dev/null +++ b/tests/modules/seqkit/replace/test.yml @@ -0,0 +1,21 @@ +- name: seqkit replace test_seqkit_replace + command: nextflow run tests/modules/seqkit/replace -entry test_seqkit_replace -c tests/config/nextflow.config + tags: + - seqkit + - seqkit/replace + files: + - path: output/seqkit/test.fasta.gz + md5sum: 053847219695c0a923d02352442d7abf + - path: output/seqkit/versions.yml + md5sum: dc9d18b7836c9db00a3032fd191bd831 + +- name: seqkit replace test_seqkit_replace_uncomp + command: nextflow run tests/modules/seqkit/replace -entry test_seqkit_replace_uncomp -c tests/config/nextflow.config + tags: + - seqkit + - seqkit/replace + files: + - path: output/seqkit/test..fasta + md5sum: 05d3294a62c72f5489f067c1da3c2f6c + - path: output/seqkit/versions.yml + md5sum: 3b88128487ec949f0bdeecebc375c407 From 62da45b0e1202677a07e2da0ee9f6181466232fb Mon Sep 17 00:00:00 2001 From: Jose Espinosa-Carrasco Date: Thu, 10 Mar 2022 09:23:45 +0100 Subject: [PATCH 9/9] Bump chromap version 0.2.0 (#1374) * Bump chromap version 0.2.0 * Temporary use the docker container until singularity container becomes available * Temporary use the docker container until singularity container available * Remove empty lines * Update singularity container after became available --- modules/chromap/chromap/main.nf | 6 +++--- modules/chromap/index/main.nf | 7 +++---- tests/modules/chromap/chromap/test.yml | 8 +++++++- tests/modules/chromap/index/test.yml | 2 ++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/modules/chromap/chromap/main.nf b/modules/chromap/chromap/main.nf index cdbf6049..4ee86b92 100644 --- a/modules/chromap/chromap/main.nf +++ b/modules/chromap/chromap/main.nf @@ -2,10 +2,10 @@ process CHROMAP_CHROMAP { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::chromap=0.1.5 bioconda::samtools=1.14" : null) + conda (params.enable_conda ? "bioconda::chromap=0.2.0 bioconda::samtools=1.14" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:724a1037d59f6a19c9d4e7bdba77b52b37de0dc3-0' : - 'quay.io/biocontainers/mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:724a1037d59f6a19c9d4e7bdba77b52b37de0dc3-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:ed3529ef5253d7ccbc688b6a4c5c447152685757-0' : + 'quay.io/biocontainers/mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:ed3529ef5253d7ccbc688b6a4c5c447152685757-0' }" input: tuple val(meta), path(reads) diff --git a/modules/chromap/index/main.nf b/modules/chromap/index/main.nf index 52deaf06..2696d6a5 100644 --- a/modules/chromap/index/main.nf +++ b/modules/chromap/index/main.nf @@ -2,11 +2,10 @@ process CHROMAP_INDEX { tag '$fasta' label 'process_medium' - conda (params.enable_conda ? "bioconda::chromap=0.1.5" : null) + conda (params.enable_conda ? "bioconda::chromap=0.2.0" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/chromap:0.1.5--h9a82719_0' : - 'quay.io/biocontainers/chromap:0.1.5--h9a82719_0' }" - + 'https://depot.galaxyproject.org/singularity/chromap:0.2.0--hd03093a_1' : + 'quay.io/biocontainers/chromap:0.2.0--hd03093a_1' }" input: path fasta diff --git a/tests/modules/chromap/chromap/test.yml b/tests/modules/chromap/chromap/test.yml index 09e5e753..40e45959 100644 --- a/tests/modules/chromap/chromap/test.yml +++ b/tests/modules/chromap/chromap/test.yml @@ -7,6 +7,8 @@ - path: output/chromap/genome.index - path: output/chromap/test.bed.gz md5sum: 25e40bde24c7b447292cd68573728694 + - path: output/chromap/versions.yml + md5sum: 2d3d2959ac20d98036807964896829e7 - name: chromap chromap test_chromap_chromap_paired_end command: nextflow run ./tests/modules/chromap/chromap -entry test_chromap_chromap_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/chromap/chromap/nextflow.config @@ -17,6 +19,8 @@ - path: output/chromap/genome.index - path: output/chromap/test.bed.gz md5sum: 7cdc8448882b75811e0c784f5f20aef2 + - path: output/chromap/versions.yml + md5sum: 51cff66779161d8a602cce5989017395 - name: chromap chromap test_chromap_chromap_paired_bam command: nextflow run ./tests/modules/chromap/chromap -entry test_chromap_chromap_paired_bam -c ./tests/config/nextflow.config -c ./tests/modules/chromap/chromap/nextflow.config @@ -26,4 +30,6 @@ files: - path: output/chromap/genome.index - path: output/chromap/test.bam - md5sum: 73e2c76007e3c61df625668e01b3f42f + md5sum: f255c7441d5a1f307fc642d2aa19647e + - path: output/chromap/versions.yml + md5sum: f91910c44169549c3923931de5c3afcb diff --git a/tests/modules/chromap/index/test.yml b/tests/modules/chromap/index/test.yml index dde1aa1b..b2aa37d8 100644 --- a/tests/modules/chromap/index/test.yml +++ b/tests/modules/chromap/index/test.yml @@ -5,3 +5,5 @@ - chromap files: - path: output/chromap/genome.index + - path: output/chromap/versions.yml + md5sum: b75dec647f9dc5f4887f36d1db7a9ccd