From c40a9a00113c3e178bf20a4f1e0870d0994d9383 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Tue, 5 Apr 2022 12:13:45 +0200 Subject: [PATCH 1/5] refactor: avoid publishing unpacked databases --- conf/modules.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 2a8789f..22db8d0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -35,11 +35,7 @@ process { } withName: UNTAR { - publishDir = [ - path: { "${params.outdir}/databases" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] + publishDir = null } withName: FASTQC { From af12435fa5e3adf240d48bf9f764a063c62593b8 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Tue, 5 Apr 2022 12:16:26 +0200 Subject: [PATCH 2/5] fix: use parameter for publish mode everywhere --- conf/modules.config | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 22db8d0..a910b8e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -43,7 +43,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}_raw" } publishDir = [ path: { "${params.outdir}/fastqc/raw" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.html' ] } @@ -53,7 +53,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}_processed" } publishDir = [ path: { "${params.outdir}/fastqc/processed" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.html' ] } @@ -69,7 +69,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}" } publishDir = [ path: { "${params.outdir}/fastp" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.fastq.gz' ] } @@ -88,7 +88,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}" } publishDir = [ path: { "${params.outdir}/fastp" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.fastq.gz' ] } @@ -104,7 +104,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}" } publishDir = [ path: { "${params.outdir}/adapterremoval" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.fastq.gz' ] } @@ -123,7 +123,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}" } publishDir = [ path: { "${params.outdir}/adapterremoval" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.fastq.gz' ] } @@ -132,7 +132,7 @@ process { ext.prefix = { "${meta.id}_${meta.run_accession}" } publishDir = [ path: { "${params.outdir}/porechop" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.fastq.gz' ] } @@ -169,7 +169,7 @@ process { ext.prefix = { "${meta.id}-${meta.run_accession}-${meta.db_name}" } publishDir = [ path: { "${params.outdir}/malt/${meta.db_name}" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.{rma6,tab,text,sam,log}' ] } @@ -179,7 +179,7 @@ process { ext.prefix = { "${meta.id}-${meta.run_accession}-${meta.db_name}" } publishDir = [ path: { "${params.outdir}/kraken2/${meta.db_name}" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*.{fastq.gz,txt}' ] } From 8827b143a22b07c21c7cbe5776fb112015f5ae8b Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Tue, 5 Apr 2022 12:30:06 +0200 Subject: [PATCH 3/5] fix: disable publishing correctly --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index a910b8e..ce9597b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -35,7 +35,7 @@ process { } withName: UNTAR { - publishDir = null + publishDir = [enabled: false] } withName: FASTQC { From 018b9a8ea60afd0c5d123e4420fdccaecafbc9f7 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 6 Apr 2022 10:21:43 +0200 Subject: [PATCH 4/5] refactor: do not generally publish output --- conf/modules.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index ce9597b..f9cfe3b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -12,12 +12,6 @@ process { - publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - withName: SAMPLESHEET_CHECK { publishDir = [ path: { "${params.outdir}/pipeline_info" }, From 7ab3aa546b8e1d0388981757d259284bf67f5fc8 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 6 Apr 2022 10:27:11 +0200 Subject: [PATCH 5/5] refactor: use flags to publish reads --- conf/modules.config | 25 ++++++++++++++----------- nextflow.config | 2 ++ nextflow_schema.json | 8 ++++++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index f9cfe3b..b59850f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -28,10 +28,6 @@ process { ] } - withName: UNTAR { - publishDir = [enabled: false] - } - withName: FASTQC { ext.args = '--quiet' ext.prefix = { "${meta.id}_${meta.run_accession}_raw" } @@ -64,7 +60,8 @@ process { publishDir = [ path: { "${params.outdir}/fastp" }, mode: params.publish_dir_mode, - pattern: '*.fastq.gz' + pattern: '*.fastq.gz', + enabled: params.save_preprocessed_reads ] } @@ -83,7 +80,8 @@ process { publishDir = [ path: { "${params.outdir}/fastp" }, mode: params.publish_dir_mode, - pattern: '*.fastq.gz' + pattern: '*.fastq.gz', + enabled: params.save_preprocessed_reads ] } @@ -99,7 +97,8 @@ process { publishDir = [ path: { "${params.outdir}/adapterremoval" }, mode: params.publish_dir_mode, - pattern: '*.fastq.gz' + pattern: '*.fastq.gz', + enabled: params.save_preprocessed_reads ] } @@ -118,7 +117,8 @@ process { publishDir = [ path: { "${params.outdir}/adapterremoval" }, mode: params.publish_dir_mode, - pattern: '*.fastq.gz' + pattern: '*.fastq.gz', + enabled: params.save_preprocessed_reads ] } @@ -127,7 +127,8 @@ process { publishDir = [ path: { "${params.outdir}/porechop" }, mode: params.publish_dir_mode, - pattern: '*.fastq.gz' + pattern: '*.fastq.gz', + enabled: params.save_preprocessed_reads ] } @@ -141,7 +142,8 @@ process { publishDir = [ path: { "${params.outdir}/bbduk/" }, mode: params.publish_dir_mode, - pattern: '*.{fastq.gz,log}' + pattern: '*.{fastq.gz,log}', + enabled: params.save_complexityfiltered_reads ] } @@ -154,7 +156,8 @@ process { publishDir = [ path: { "${params.outdir}/prinseqplusplus/" }, mode: params.publish_dir_mode, - pattern: '*{_good_out.fastq.gz,_good_out_R1.fastq.gz,_good_out_R2.fastq.gz,log}' + pattern: '*{_good_out.fastq.gz,_good_out_R1.fastq.gz,_good_out_R2.fastq.gz,log}', + enabled: params.save_complexityfiltered_reads ] } diff --git a/nextflow.config b/nextflow.config index e6d6307..19cc823 100644 --- a/nextflow.config +++ b/nextflow.config @@ -64,6 +64,7 @@ params { shortread_clipmerge_adapter2 = null shortread_clipmerge_minlength = 15 longread_clip = false + save_preprocessed_reads = false // Complexity filtering shortread_complexityfilter = false @@ -73,6 +74,7 @@ params { shortread_complexityfilter_bbduk_mask = false shortread_complexityfilter_prinseqplusplus_mode = 'entropy' shortread_complexityfilter_prinseqplusplus_dustscore = 0.5 + save_complexityfiltered_reads = false // MALT diff --git a/nextflow_schema.json b/nextflow_schema.json index 6f6125e..6858409 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -308,6 +308,10 @@ "type": "integer", "default": 15 }, + "save_preprocessed_reads": { + "type": "boolean", + "default": false + }, "shortread_complexityfilter_tool": { "type": "string", "default": "bbduk" @@ -334,6 +338,10 @@ "shortread_complexityfilter_prinseqplusplus_dustscore": { "type": "number", "default": 0.5 + }, + "save_complexityfiltered_reads": { + "type": "boolean", + "default": false } } }