From 3ae8cbfad754dc45cae7fb42315f2873a7e9e417 Mon Sep 17 00:00:00 2001 From: CMGG ICT Team Date: Wed, 11 May 2022 13:32:56 +0200 Subject: [PATCH 01/10] bowtie2: remove code duplication --- modules/bowtie2/align/main.nf | 112 +++++++++++++-------------- tests/modules/bowtie2/align/test.yml | 88 ++++----------------- 2 files changed, 65 insertions(+), 135 deletions(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index c233f955..18209bdf 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -1,11 +1,11 @@ process BOWTIE2_ALIGN { tag "$meta.id" - label 'process_high' + label "process_high" - conda (params.enable_conda ? 'bioconda::bowtie2=2.4.4 bioconda::samtools=1.15.1 conda-forge::pigz=2.6' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0' : - 'quay.io/biocontainers/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0' }" + conda (params.enable_conda ? "bioconda::bowtie2=2.4.4 bioconda::samtools=1.15.1 conda-forge::pigz=2.6" : null) + container "${ workflow.containerEngine == "singularity" && !task.ext.singularity_pull_docker_container ? + "https://depot.galaxyproject.org/singularity/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0" : + "quay.io/biocontainers/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0" }" input: tuple val(meta), path(reads) @@ -13,69 +13,61 @@ process BOWTIE2_ALIGN { val save_unaligned output: - tuple val(meta), path('*.bam') , emit: bam - tuple val(meta), path('*.log') , emit: log - tuple val(meta), path('*fastq.gz'), emit: fastq, optional:true + tuple val(meta), path("*.bam") , emit: bam + tuple val(meta), path("*.log") , emit: log + tuple val(meta), path("*fastq.gz"), emit: fastq, optional:true path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' + def args = task.ext.args ?: "" + def args2 = task.ext.args2 ?: "" def prefix = task.ext.prefix ?: "${meta.id}" + + def unaligned = "" + def reads_args = "" + if (meta.single_end) { - def unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : '' - """ - INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` - [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` - [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 - bowtie2 \\ - -x \$INDEX \\ - -U $reads \\ - --threads $task.cpus \\ - $unaligned \\ - $args \\ - 2> ${prefix}.bowtie2.log \\ - | samtools view -@ $task.cpus $args2 -bhS -o ${prefix}.bam - - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//') - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) - END_VERSIONS - """ + unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : "" + reads_args = "-U ${reads}" } else { - def unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : '' - """ - INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` - [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` - [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 - bowtie2 \\ - -x \$INDEX \\ - -1 ${reads[0]} \\ - -2 ${reads[1]} \\ - --threads $task.cpus \\ - $unaligned \\ - $args \\ - 2> ${prefix}.bowtie2.log \\ - | samtools view -@ $task.cpus $args2 -bhS -o ${prefix}.bam - - - if [ -f ${prefix}.unmapped.fastq.1.gz ]; then - mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz - fi - if [ -f ${prefix}.unmapped.fastq.2.gz ]; then - mv ${prefix}.unmapped.fastq.2.gz ${prefix}.unmapped_2.fastq.gz - fi - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//') - samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') - pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) - END_VERSIONS - """ + unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : "" + reads_args = "-1 ${reads[0]} -2 ${reads[1]}" } + + def samtools = "samtools view -@ $task.cpus --fast --with-header ${args2} > ${prefix}.bam" + + + """ + INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/.rev.1.bt2//"` + [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed "s/.rev.1.bt2l//"` + [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 + + bowtie2 \\ + -x \$INDEX \\ + ${reads_args} \\ + --threads $task.cpus \\ + ${unaligned} \\ + $args \\ + 2> ${prefix}.bowtie2.log \\ + | ${samtools} + + if [ -f ${prefix}.unmapped.fastq.1.gz ]; then + mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz + fi + + if [ -f ${prefix}.unmapped.fastq.2.gz ]; then + mv ${prefix}.unmapped.fastq.2.gz ${prefix}.unmapped_2.fastq.gz + fi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//') + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ } + diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index ea9aa72a..b3cb8330 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -1,83 +1,21 @@ -- name: bowtie2 align single-end - command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config +- name: bowtie2 align test_bowtie2_align_single_end + command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c tests/config/nextflow.config tags: - - bowtie2 - bowtie2/align + - bowtie2 files: - - path: ./output/bowtie2/test.bam - - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2 - md5sum: 4ed93abba181d8dfab2e303e33114777 - - path: ./output/bowtie2/bowtie2/genome.2.bt2 - md5sum: 47b153cd1319abc88dda532462651fcf - - path: ./output/bowtie2/bowtie2/genome.1.bt2 - md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf - - path: ./output/bowtie2/bowtie2/genome.4.bt2 - md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 - md5sum: 52be6950579598a990570fbcf5372184 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 - md5sum: e3b4ef343dea4dd571642010a7d09597 + - path: output/bowtie2/test.bam + - path: output/bowtie2/test.bowtie2.log + md5sum: 7b8a9e61b7646da1089b041333c41a87 + - path: output/bowtie2/versions.yml -- name: bowtie2 align paired-end - command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config +- name: bowtie2 align test_bowtie2_align_paired_end + command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c tests/config/nextflow.config tags: - - bowtie2 - bowtie2/align - files: - - path: ./output/bowtie2/test.bam - - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2 - md5sum: 4ed93abba181d8dfab2e303e33114777 - - path: ./output/bowtie2/bowtie2/genome.2.bt2 - md5sum: 47b153cd1319abc88dda532462651fcf - - path: ./output/bowtie2/bowtie2/genome.1.bt2 - md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf - - path: ./output/bowtie2/bowtie2/genome.4.bt2 - md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 - md5sum: 52be6950579598a990570fbcf5372184 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 - md5sum: e3b4ef343dea4dd571642010a7d09597 - -- name: bowtie2 align single-end large-index - command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index - tags: - bowtie2 - - bowtie2/align files: - - path: ./output/bowtie2/test.bam - - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2l - md5sum: 8952b3e0b1ce9a7a5916f2e147180853 - - path: ./output/bowtie2/bowtie2/genome.2.bt2l - md5sum: 22c284084784a0720989595e0c9461fd - - path: ./output/bowtie2/bowtie2/genome.1.bt2l - md5sum: 07d811cd4e350d56267183d2ac7023a5 - - path: ./output/bowtie2/bowtie2/genome.4.bt2l - md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l - md5sum: fda48e35925fb24d1c0785f021981e25 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l - md5sum: 802c26d32b970e1b105032b7ce7348b4 - -- name: bowtie2 align paired-end large-index - command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index - tags: - - bowtie2 - - bowtie2/align - files: - - path: ./output/bowtie2/test.bam - - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2l - md5sum: 8952b3e0b1ce9a7a5916f2e147180853 - - path: ./output/bowtie2/bowtie2/genome.2.bt2l - md5sum: 22c284084784a0720989595e0c9461fd - - path: ./output/bowtie2/bowtie2/genome.1.bt2l - md5sum: 07d811cd4e350d56267183d2ac7023a5 - - path: ./output/bowtie2/bowtie2/genome.4.bt2l - md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l - md5sum: fda48e35925fb24d1c0785f021981e25 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l - md5sum: 802c26d32b970e1b105032b7ce7348b4 + - path: output/bowtie2/test.bam + - path: output/bowtie2/test.bowtie2.log + md5sum: bd89ce1b28c93bf822bae391ffcedd19 + - path: output/bowtie2/versions.yml From 0d00452ecaff35d83ce931f57be9ab6244245c36 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:55:45 +0200 Subject: [PATCH 02/10] Update tests/modules/bowtie2/align/test.yml Co-authored-by: Harshil Patel --- tests/modules/bowtie2/align/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index b3cb8330..9aec709c 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -1,5 +1,5 @@ - name: bowtie2 align test_bowtie2_align_single_end - command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config tags: - bowtie2/align - bowtie2 From 9dd083fa9480fd03b4995e364d2634e78b572049 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:55:55 +0200 Subject: [PATCH 03/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 18209bdf..605aae34 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -28,7 +28,6 @@ process BOWTIE2_ALIGN { def unaligned = "" def reads_args = "" - if (meta.single_end) { unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : "" reads_args = "-U ${reads}" From 94e8c3a882b7c03e7c24ee9a710a28511e31611f Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:56:03 +0200 Subject: [PATCH 04/10] Update tests/modules/bowtie2/align/test.yml Co-authored-by: Harshil Patel --- tests/modules/bowtie2/align/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 9aec709c..ef05d70d 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -10,7 +10,7 @@ - path: output/bowtie2/versions.yml - name: bowtie2 align test_bowtie2_align_paired_end - command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config tags: - bowtie2/align - bowtie2 From ff4b297827d3a78ea66172241a4ed414ea1148da Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:57:11 +0200 Subject: [PATCH 05/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 605aae34..8ee198bb 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -48,7 +48,7 @@ process BOWTIE2_ALIGN { -x \$INDEX \\ ${reads_args} \\ --threads $task.cpus \\ - ${unaligned} \\ + $unaligned \\ $args \\ 2> ${prefix}.bowtie2.log \\ | ${samtools} From b456d8fa0aa797258ea9a308120be45e5b7b9667 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:57:19 +0200 Subject: [PATCH 06/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 8ee198bb..b7334614 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -46,7 +46,7 @@ process BOWTIE2_ALIGN { bowtie2 \\ -x \$INDEX \\ - ${reads_args} \\ + $reads_args \\ --threads $task.cpus \\ $unaligned \\ $args \\ From 971a889e2a8adce2c8831f296ac37d335c5f93c7 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:57:27 +0200 Subject: [PATCH 07/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index b7334614..6d65aa50 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -42,7 +42,7 @@ process BOWTIE2_ALIGN { """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/.rev.1.bt2//"` [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed "s/.rev.1.bt2l//"` - [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 + [ -z "\$INDEX" ] && echo "Bowtie2 index files not found" 1>&2 && exit 1 bowtie2 \\ -x \$INDEX \\ From 9097abf277f25b34bd41480c35bd7deac6f9a7c2 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:58:58 +0200 Subject: [PATCH 08/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 6d65aa50..126c93ae 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -36,7 +36,7 @@ process BOWTIE2_ALIGN { reads_args = "-1 ${reads[0]} -2 ${reads[1]}" } - def samtools = "samtools view -@ $task.cpus --fast --with-header ${args2} > ${prefix}.bam" + def samtools_command = "samtools view -@ $task.cpus --fast --with-header ${args2} > ${prefix}.bam" """ From bcd7fe55b171e4532cd19312c091258ee7ccc26a Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 13:59:03 +0200 Subject: [PATCH 09/10] Update modules/bowtie2/align/main.nf Co-authored-by: Harshil Patel --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 126c93ae..9fd19469 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -51,7 +51,7 @@ process BOWTIE2_ALIGN { $unaligned \\ $args \\ 2> ${prefix}.bowtie2.log \\ - | ${samtools} + | $samtools_command if [ -f ${prefix}.unmapped.fastq.1.gz ]; then mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz From c8d3de2ad965bfd8e165f6f6d91249eea3436854 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 11 May 2022 16:17:30 +0200 Subject: [PATCH 10/10] restore samtools cmd to defaults --- modules/bowtie2/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 9fd19469..38242fcb 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -36,7 +36,7 @@ process BOWTIE2_ALIGN { reads_args = "-1 ${reads[0]} -2 ${reads[1]}" } - def samtools_command = "samtools view -@ $task.cpus --fast --with-header ${args2} > ${prefix}.bam" + def samtools_command = "samtools view -@ $task.cpus --bam --with-header ${args2} > ${prefix}.bam" """