From 5a49d2c1bfdaa1204b83b172c3ca95cf972402b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Guizard?= Date: Thu, 7 Oct 2021 10:06:02 +0100 Subject: [PATCH] New module: `isoseq3/cluster` (#801) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * πŸ“¦ NEW: Add isoseq3/cluster module * πŸ›FIX: Fix reports channel and add .pbi to it * πŸ›FIX: Fix report channel definition * πŸ‘ŒIMPROVE: Move .pbi file into reports channel * πŸ‘ŒIMPROVE: remove --use_qvs option from command line * πŸ‘Œ IMPROVE: Add in addParams removed options from command line * πŸ‘Œ IMPROVE: Add some pacbio test files * πŸ› FIX: Add Pacbio index to test_data.config * πŸ‘Œ IMPROVE: Re add 10000 data test * πŸ‘Œ IMPROVE: Add some pbindex * πŸ› FIX: Add pbi extension to files * πŸ‘Œ IMPROVE: The module accept one channel (primers moved into the first channel) * πŸ“¦ NEW: Add galgal6 chr30 test data * πŸ“¦ NEW: Add bamtools module * πŸ‘Œ IMPROVE: Ignore test data * πŸ‘Œ IMPROVE: Update to last templates version * πŸ‘Œ IMPROVE: Update to last templates version * πŸ‘Œ IMPROVE: add singletons parameter and improve outputs * πŸ› FIX: Update test with last module model * πŸ‘Œ IMPROVE: Add test tag * πŸ‘Œ IMPROVE: Update module to last template version * πŸ‘Œ IMPROVE: Update module to last template version * πŸ‘Œ IMPROVE: Update test data config * πŸ‘Œ IMPROVE: Remove pbi from input files * πŸ‘Œ IMPROVE: Remove unused index * πŸ‘Œ IMPROVE: Final version of test datasets config * πŸ‘Œ IMPROVE: Final version of test datasets config * πŸ‘Œ IMPROVE: Remove useless index + Fix Typos * πŸ‘Œ IMPROVE: Remove useless index + Fix Typos * πŸ‘Œ IMPROVE: Fill contains args * πŸ“¦ NEW: Add isoseq3/cluster module * πŸ›FIX: Fix reports channel and add .pbi to it * πŸ›FIX: Fix report channel definition * πŸ‘ŒIMPROVE: Move .pbi file into reports channel * πŸ‘ŒIMPROVE: remove --use_qvs option from command line * πŸ‘Œ IMPROVE: Add in addParams removed options from command line * πŸ‘Œ IMPROVE: Add some pacbio test files * πŸ› FIX: Add Pacbio index to test_data.config * πŸ‘Œ IMPROVE: Re add 10000 data test * πŸ‘Œ IMPROVE: Add some pbindex * πŸ› FIX: Add pbi extension to files * πŸ‘Œ IMPROVE: The module accept one channel (primers moved into the first channel) * πŸ‘Œ IMPROVE: Update to last templates version * πŸ‘Œ IMPROVE: add singletons parameter and improve outputs * πŸ› FIX: Update test with last module model * πŸ‘Œ IMPROVE: Add test tag * πŸ‘Œ IMPROVE: Update test data config * πŸ‘Œ IMPROVE: Remove pbi from input files * πŸ‘Œ IMPROVE: Remove unused index * πŸ‘Œ IMPROVE: Final version of test datasets config * πŸ‘Œ IMPROVE: Remove useless index + Fix Typos * πŸ‘Œ IMPROVE: Fill contains args * πŸ‘Œ IMPROVE: Add some pacbio test files * πŸ› FIX: Add Pacbio index to test_data.config * πŸ‘Œ IMPROVE: Re add 10000 data test * πŸ‘Œ IMPROVE: Add some pbindex * πŸ› FIX: Add pbi extension to files * πŸ“¦ NEW: Add galgal6 chr30 test data * πŸ“¦ NEW: Add bamtools module * πŸ‘Œ IMPROVE: Ignore test data * πŸ‘Œ IMPROVE: Update to last templates version * πŸ‘Œ IMPROVE: Update module to last template version * πŸ‘Œ IMPROVE: Update module to last template version * πŸ‘Œ IMPROVE: Final version of test datasets config * πŸ‘Œ IMPROVE: Remove useless index + Fix Typos * πŸ‘Œ IMPROVE: Update code to new versions capture + better output channels * πŸ‘Œ IMPROVE: Update with new versions.yml file * πŸ› FIX: Update meta.yml + correct typos * πŸ‘Œ IMPROVE: Clean output file names + correct typo * πŸ› FIX: Remove bamtools/split module from isoseq3/cluster --- .gitignore | 4 +- modules/isoseq3/cluster/functions.nf | 78 +++++++++++++++++++++++++ modules/isoseq3/cluster/main.nf | 53 +++++++++++++++++ modules/isoseq3/cluster/meta.yml | 81 ++++++++++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/isoseq3/cluster/main.nf | 15 +++++ tests/modules/isoseq3/cluster/test.yml | 28 +++++++++ 7 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 modules/isoseq3/cluster/functions.nf create mode 100644 modules/isoseq3/cluster/main.nf create mode 100644 modules/isoseq3/cluster/meta.yml create mode 100644 tests/modules/isoseq3/cluster/main.nf create mode 100644 tests/modules/isoseq3/cluster/test.yml diff --git a/.gitignore b/.gitignore index 06eae014..c773a2d0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,9 @@ test_output/ output/ .DS_Store *.code-workspace +tests/data/ .screenrc .*.sw? __pycache__ *.pyo *.pyc -tests/data/ -modules/modtest/ -tests/modules/modtest/ diff --git a/modules/isoseq3/cluster/functions.nf b/modules/isoseq3/cluster/functions.nf new file mode 100644 index 00000000..85628ee0 --- /dev/null +++ b/modules/isoseq3/cluster/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/isoseq3/cluster/main.nf b/modules/isoseq3/cluster/main.nf new file mode 100644 index 00000000..f01af2bc --- /dev/null +++ b/modules/isoseq3/cluster/main.nf @@ -0,0 +1,53 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process ISOSEQ3_CLUSTER { + tag "$meta.id" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::isoseq3=3.4.0" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/isoseq3:3.4.0--0" + } else { + container "quay.io/biocontainers/isoseq3:3.4.0--0" + } + + input: + tuple val(meta), path(bam) + + output: + tuple val(meta), path("*.bam") , emit: bam + tuple val(meta), path("*.bam.pbi") , emit: pbi + tuple val(meta), path("*.cluster") , emit: cluster + tuple val(meta), path("*.cluster_report.csv"), emit: cluster_report + tuple val(meta), path("*.transcriptset.xml") , emit: transcriptset + tuple val(meta), path("*.hq.bam") , emit: hq_bam + tuple val(meta), path("*.hq.bam.pbi") , emit: hq_pbi + tuple val(meta), path("*.lq.bam") , emit: lq_bam + tuple val(meta), path("*.lq.bam.pbi") , emit: lq_pbi + path "versions.yml" , emit: versions + + tuple val(meta), path("*.singletons.bam") , optional: true, emit: singletons_bam + tuple val(meta), path("*.singletons.bam.pbi"), optional: true, emit: singletons_pbi + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + isoseq3 \\ + cluster \\ + $bam \\ + ${prefix}.bam \\ + $options.args + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + isoseq3 cluster: \$( isoseq3 cluster --version|sed 's/isoseq cluster //g'|sed 's/ (.*//g' ) + END_VERSIONS + """ +} diff --git a/modules/isoseq3/cluster/meta.yml b/modules/isoseq3/cluster/meta.yml new file mode 100644 index 00000000..6fadb9c4 --- /dev/null +++ b/modules/isoseq3/cluster/meta.yml @@ -0,0 +1,81 @@ +name: isoseq3_cluster +description: IsoSeq3 - Cluster - Cluster trimmed consensus sequences +keywords: + - cluster +tools: + - isoseq3: + description: IsoSeq3 - Cluster - Cluster trimmed consensus sequences + homepage: https://github.com/PacificBiosciences/IsoSeq/blob/master/isoseq-clustering.md + documentation: https://github.com/PacificBiosciences/IsoSeq/blob/master/isoseq-clustering.md + tool_dev_url: https://github.com/PacificBiosciences/IsoSeq/blob/master/isoseq-clustering.md + doi: "" + licence: ['BSD-3-clause-Clear'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: BAM file generated by isoseq3 refine + pattern: "*.bam" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - version: + type: file + description: File containing software version + pattern: "versions.yml" + - bam: + type: file + description: BAM file of clustered consensus + pattern: "*.bam" + - pbi: + type: file + description: Pacbio Index of consensus reads generated by clustering + pattern: "*.pbi" + - cluster: + type: file + description: A two columns (from, to) file describing original read name to new read name + pattern: "*.cluster" + - cluster_report: + type: file + description: A table files clusters (transcripts) members (read) + pattern: "*.cluster_report.csv" + - transcriptset: + type: file + description: A metadata xml file which contains full paths to data files + pattern: "*.clustered.transcriptset.xml" + - hq_bam: + type: file + description: High quality reads + pattern: "*.hq.bam" + - hq_pbi: + type: file + description: Pacbio index of high quality reads + pattern: "*.hq.bam.pbi" + - lq_bam: + type: file + description: Low quality reads + pattern: "*.lq.bam" + - lq_pbi: + type: file + description: Pacbio index of low quality reads + pattern: "*.lq.bam.pbi" + - singletons_bam: + type: file + description: Unclustered reads + pattern: "*.singletons.bam" + - singletons_pbi: + type: file + description: Pacbio index of unclustered reads + pattern: "*.singletons.bam.pbi" + +authors: + - "@sguizard" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index bf280360..6f0e491f 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -525,6 +525,10 @@ iqtree: - modules/iqtree/** - tests/modules/iqtree/** +isoseq3/cluster: + - modules/isoseq3/cluster/** + - tests/modules/isoseq3/cluster/** + isoseq3/refine: - modules/isoseq3/refine/** - tests/modules/isoseq3/refine/** diff --git a/tests/modules/isoseq3/cluster/main.nf b/tests/modules/isoseq3/cluster/main.nf new file mode 100644 index 00000000..90a24c11 --- /dev/null +++ b/tests/modules/isoseq3/cluster/main.nf @@ -0,0 +1,15 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { ISOSEQ3_CLUSTER } from '../../../../modules/isoseq3/cluster/main.nf' addParams( options: [args: '--singletons --use-qvs --verbose'] ) + +workflow test_isoseq3_cluster { + + input = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['pacbio']['refine'], checkIfExists: true), + ] + + ISOSEQ3_CLUSTER ( input ) +} diff --git a/tests/modules/isoseq3/cluster/test.yml b/tests/modules/isoseq3/cluster/test.yml new file mode 100644 index 00000000..cc6b6dac --- /dev/null +++ b/tests/modules/isoseq3/cluster/test.yml @@ -0,0 +1,28 @@ +- name: isoseq3 cluster test_isoseq3_cluster + command: nextflow run tests/modules/isoseq3/cluster -entry test_isoseq3_cluster -c tests/config/nextflow.config + tags: + - isoseq3 + - isoseq3/cluster + files: + - path: output/isoseq3/test.bam + md5sum: ca8277f4d8fe1bba68ba266c42b46dd1 + - path: output/isoseq3/test.bam.pbi + md5sum: cbc06657b4543faba7ff886b3b12b862 + - path: output/isoseq3/test.cluster + md5sum: d5059d856763fc5591332980bfc0d57b + - path: output/isoseq3/test.cluster_report.csv + md5sum: 342d97dc10aedf80a45977edcb491c62 + - path: output/isoseq3/test.hq.bam + md5sum: e93ea85776c35c246364d954032c2ea9 + - path: output/isoseq3/test.hq.bam.pbi + md5sum: 5a8ea7668e8f8e173478b28cbb6ab515 + - path: output/isoseq3/test.lq.bam + md5sum: 4ea0e4f4a6cc689dcc275adcdf688fad + - path: output/isoseq3/test.lq.bam.pbi + md5sum: f5edc24711b2c8d6474d60cb69022af0 + - path: output/isoseq3/test.singletons.bam + md5sum: 73d131920bd42e1fc5fca2e6cb71f4b2 + - path: output/isoseq3/test.singletons.bam.pbi + md5sum: 73980863be4b5bda2846325c737f0b5e + - path: output/isoseq3/test.transcriptset.xml + contains: [ 'PacBio.DataSet.TranscriptSet' ]