From 46a9aabfa3f41a70b21def6b441d8759aa3870d2 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 8 May 2022 16:05:25 +0200 Subject: [PATCH 1/7] chore: improve license description --- modules/sratools/prefetch/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sratools/prefetch/meta.yml b/modules/sratools/prefetch/meta.yml index e08b708c..2ca53af3 100644 --- a/modules/sratools/prefetch/meta.yml +++ b/modules/sratools/prefetch/meta.yml @@ -10,7 +10,7 @@ tools: homepage: https://github.com/ncbi/sra-tools documentation: https://github.com/ncbi/sra-tools/wiki tool_dev_url: https://github.com/ncbi/sra-tools - licence: ["US-Government-Work"] + licence: ["Public Domain"] input: - meta: From f6936e5270477bd3c62a54c17ed2966b61c1705a Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 8 May 2022 16:05:51 +0200 Subject: [PATCH 2/7] refactor: accept settings as input --- modules/sratools/prefetch/main.nf | 5 ++--- modules/sratools/prefetch/templates/retry_with_backoff.sh | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/sratools/prefetch/main.nf b/modules/sratools/prefetch/main.nf index 3408c3e9..889fef52 100644 --- a/modules/sratools/prefetch/main.nf +++ b/modules/sratools/prefetch/main.nf @@ -9,10 +9,11 @@ process SRATOOLS_PREFETCH { input: tuple val(meta), val(id) + path(ncbi_settings) output: tuple val(meta), path(id), emit: sra - path "versions.yml" , emit: versions + path 'versions.yml' , emit: versions when: task.ext.when == null || task.ext.when @@ -20,7 +21,5 @@ process SRATOOLS_PREFETCH { shell: args = task.ext.args ?: '' args2 = task.ext.args2 ?: '5 1 100' // - config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n" - template 'retry_with_backoff.sh' } diff --git a/modules/sratools/prefetch/templates/retry_with_backoff.sh b/modules/sratools/prefetch/templates/retry_with_backoff.sh index fbcb6532..17395ef1 100755 --- a/modules/sratools/prefetch/templates/retry_with_backoff.sh +++ b/modules/sratools/prefetch/templates/retry_with_backoff.sh @@ -40,10 +40,8 @@ retry_with_backoff() { echo "${output}" } -eval "$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')" -if [[ ! -f "${NCBI_SETTINGS}" ]]; then - mkdir -p "$(dirname "${NCBI_SETTINGS}")" - printf '!{config}' > "${NCBI_SETTINGS}" +if [[ "!{ncbi_settings.name}" != "EXISTS" ]]; then + export NCBI_SETTINGS="$PWD/!{ncbi_settings}" fi retry_with_backoff !{args2} \ From 145e2f80e9165759d8a2994c415378882cdda9a2 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 8 May 2022 16:34:18 +0200 Subject: [PATCH 3/7] tests: run prefetch with and without given settings --- tests/modules/sratools/prefetch/main.nf | 25 ++++++++++++++++--- .../modules/sratools/prefetch/nextflow.config | 3 +++ .../sratools/prefetch/nextflow_mount.config | 17 +++++++++++++ tests/modules/sratools/prefetch/test.yml | 23 ++++++++++++++--- 4 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 tests/modules/sratools/prefetch/nextflow_mount.config diff --git a/tests/modules/sratools/prefetch/main.nf b/tests/modules/sratools/prefetch/main.nf index aa6252a1..eb579920 100644 --- a/tests/modules/sratools/prefetch/main.nf +++ b/tests/modules/sratools/prefetch/main.nf @@ -4,12 +4,31 @@ nextflow.enable.dsl = 2 include { SRATOOLS_PREFETCH } from '../../../../modules/sratools/prefetch/main.nf' -workflow test_sratools_prefetch { +workflow test_sratools_prefetch_with_settings_input { + + file(params.settings_path).mkdirs() + def settings = file(params.settings_file) + settings.text = "/LIBS/GUID = \"5b0d4b7d-88c7-4802-98fd-e3afd06feb32\"\n/libs/cloud/report_instance_identity = \"true\"\n" input = [ [ id:'test', single_end:false ], // meta map - 'ERR2815334' + 'DRR000774' ] - SRATOOLS_PREFETCH ( input ) + SRATOOLS_PREFETCH(input, settings) } + +workflow test_sratools_prefetch_without_settings_input { + + file(params.settings_path).mkdirs() + def settings = file(params.settings_file) + settings.text = "/LIBS/GUID = \"5b0d4b7d-88c7-4802-98fd-e3afd06feb32\"\n/libs/cloud/report_instance_identity = \"true\"\n" + + input = [ + [ id:'test', single_end:false ], // meta map + 'DRR000774' + ] + + SRATOOLS_PREFETCH(input, file('EXISTS')) +} + diff --git a/tests/modules/sratools/prefetch/nextflow.config b/tests/modules/sratools/prefetch/nextflow.config index 8730f1c4..a6c70bd3 100644 --- a/tests/modules/sratools/prefetch/nextflow.config +++ b/tests/modules/sratools/prefetch/nextflow.config @@ -1,3 +1,6 @@ +params.settings_path = '/tmp/.ncbi' +params.settings_file = "${params.settings_path}/user-settings.mkfg" + process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } diff --git a/tests/modules/sratools/prefetch/nextflow_mount.config b/tests/modules/sratools/prefetch/nextflow_mount.config new file mode 100644 index 00000000..1a0eed2a --- /dev/null +++ b/tests/modules/sratools/prefetch/nextflow_mount.config @@ -0,0 +1,17 @@ +params.settings_path = '/tmp/.ncbi' +params.settings_file = "${params.settings_path}/user-settings.mkfg" + +env.NCBI_SETTINGS = params.settings_file + +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + + withName: SRATOOLS_PREFETCH { + containerOptions = { + (workflow.containerEngine == 'singularity') ? + "-B ${params.settings_path}:${params.settings_path}" : + "-v ${params.settings_path}:${params.settings_path}" + } + } +} diff --git a/tests/modules/sratools/prefetch/test.yml b/tests/modules/sratools/prefetch/test.yml index a2efef77..d747b9a6 100644 --- a/tests/modules/sratools/prefetch/test.yml +++ b/tests/modules/sratools/prefetch/test.yml @@ -1,8 +1,23 @@ -- name: sratools prefetch test_sratools_prefetch - command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow.config +- name: sratools prefetch test_sratools_prefetch_with_settings_input + command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch_with_settings_input -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow.config tags: - sratools/prefetch - sratools files: - - path: output/sratools/ERR2815334/ERR2815334.sra - md5sum: 9a98c7f6f4774b7ef94aa915b92a54ea + - path: output/sratools/DRR000774/DRR000774.sra + md5sum: 7647dba20c89c0e3d7ad13842f060eb0 + - path: output/sratools/versions.yml + contains: + - "sratools: 2.11.0" + +- name: sratools prefetch test_sratools_prefetch_without_settings_input + command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch_without_settings_input -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow_mount.config + tags: + - sratools/prefetch + - sratools + files: + - path: output/sratools/DRR000774/DRR000774.sra + md5sum: 7647dba20c89c0e3d7ad13842f060eb0 + - path: output/sratools/versions.yml + contains: + - "sratools: 2.11.0" From b1e134fdd765b1c5dfeaa9433c1c11ba481c803e Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 8 May 2022 16:43:30 +0200 Subject: [PATCH 4/7] docs: describe new file input --- modules/sratools/prefetch/meta.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/sratools/prefetch/meta.yml b/modules/sratools/prefetch/meta.yml index 2ca53af3..f8c50a0d 100644 --- a/modules/sratools/prefetch/meta.yml +++ b/modules/sratools/prefetch/meta.yml @@ -22,6 +22,12 @@ input: type: val description: > A string denoting an SRA id. + - ncbi_settings: + type: file + description: > + Either a proper NCBI settings file or in case an existing file should be used, + a file with name EXISTS. + pattern: "*.mkfg | EXISTS" output: - meta: From 09bfe3c6182addc683d15620fa7d07db42b38946 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 9 May 2022 12:28:58 +0200 Subject: [PATCH 5/7] Apply suggestions from code review Co-authored-by: Harshil Patel --- modules/sratools/prefetch/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sratools/prefetch/main.nf b/modules/sratools/prefetch/main.nf index 889fef52..3426ae12 100644 --- a/modules/sratools/prefetch/main.nf +++ b/modules/sratools/prefetch/main.nf @@ -9,7 +9,7 @@ process SRATOOLS_PREFETCH { input: tuple val(meta), val(id) - path(ncbi_settings) + path ncbi_settings output: tuple val(meta), path(id), emit: sra From a303904eb00c814132c5192c86ab6a1aa7f7cc17 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Tue, 10 May 2022 12:19:25 +0200 Subject: [PATCH 6/7] chore: add settings file to test data config --- tests/config/test_data.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/config/test_data.config b/tests/config/test_data.config index c4f470a4..7082c5ab 100644 --- a/tests/config/test_data.config +++ b/tests/config/test_data.config @@ -422,6 +422,9 @@ params { test_merge_cool_cp2 = "${test_data_dir}/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cp2.cool" } + 'config' { + ncbi_user_settings = "${test_data_dir}/generic/config/ncbi_user_settings.mkfg" + } } } } From 1b228835e9525990db99243cb4f0d07aa6e01bc3 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Tue, 10 May 2022 12:29:42 +0200 Subject: [PATCH 7/7] refactor: require settings input always --- modules/sratools/prefetch/meta.yml | 5 ++-- .../prefetch/templates/retry_with_backoff.sh | 4 +--- tests/modules/sratools/prefetch/main.nf | 23 ++----------------- .../modules/sratools/prefetch/nextflow.config | 3 --- .../sratools/prefetch/nextflow_mount.config | 17 -------------- tests/modules/sratools/prefetch/test.yml | 16 ++----------- 6 files changed, 7 insertions(+), 61 deletions(-) delete mode 100644 tests/modules/sratools/prefetch/nextflow_mount.config diff --git a/modules/sratools/prefetch/meta.yml b/modules/sratools/prefetch/meta.yml index f8c50a0d..a3a26522 100644 --- a/modules/sratools/prefetch/meta.yml +++ b/modules/sratools/prefetch/meta.yml @@ -25,9 +25,8 @@ input: - ncbi_settings: type: file description: > - Either a proper NCBI settings file or in case an existing file should be used, - a file with name EXISTS. - pattern: "*.mkfg | EXISTS" + An NCBI user settings file. + pattern: "*.mkfg" output: - meta: diff --git a/modules/sratools/prefetch/templates/retry_with_backoff.sh b/modules/sratools/prefetch/templates/retry_with_backoff.sh index 17395ef1..cec0ab43 100755 --- a/modules/sratools/prefetch/templates/retry_with_backoff.sh +++ b/modules/sratools/prefetch/templates/retry_with_backoff.sh @@ -40,9 +40,7 @@ retry_with_backoff() { echo "${output}" } -if [[ "!{ncbi_settings.name}" != "EXISTS" ]]; then - export NCBI_SETTINGS="$PWD/!{ncbi_settings}" -fi +export NCBI_SETTINGS="$PWD/!{ncbi_settings}" retry_with_backoff !{args2} \ prefetch \ diff --git a/tests/modules/sratools/prefetch/main.nf b/tests/modules/sratools/prefetch/main.nf index eb579920..cf9794ab 100644 --- a/tests/modules/sratools/prefetch/main.nf +++ b/tests/modules/sratools/prefetch/main.nf @@ -4,31 +4,12 @@ nextflow.enable.dsl = 2 include { SRATOOLS_PREFETCH } from '../../../../modules/sratools/prefetch/main.nf' -workflow test_sratools_prefetch_with_settings_input { - - file(params.settings_path).mkdirs() - def settings = file(params.settings_file) - settings.text = "/LIBS/GUID = \"5b0d4b7d-88c7-4802-98fd-e3afd06feb32\"\n/libs/cloud/report_instance_identity = \"true\"\n" +workflow test_sratools_prefetch { input = [ [ id:'test', single_end:false ], // meta map 'DRR000774' ] - SRATOOLS_PREFETCH(input, settings) + SRATOOLS_PREFETCH(input, file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true)) } - -workflow test_sratools_prefetch_without_settings_input { - - file(params.settings_path).mkdirs() - def settings = file(params.settings_file) - settings.text = "/LIBS/GUID = \"5b0d4b7d-88c7-4802-98fd-e3afd06feb32\"\n/libs/cloud/report_instance_identity = \"true\"\n" - - input = [ - [ id:'test', single_end:false ], // meta map - 'DRR000774' - ] - - SRATOOLS_PREFETCH(input, file('EXISTS')) -} - diff --git a/tests/modules/sratools/prefetch/nextflow.config b/tests/modules/sratools/prefetch/nextflow.config index a6c70bd3..8730f1c4 100644 --- a/tests/modules/sratools/prefetch/nextflow.config +++ b/tests/modules/sratools/prefetch/nextflow.config @@ -1,6 +1,3 @@ -params.settings_path = '/tmp/.ncbi' -params.settings_file = "${params.settings_path}/user-settings.mkfg" - process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } diff --git a/tests/modules/sratools/prefetch/nextflow_mount.config b/tests/modules/sratools/prefetch/nextflow_mount.config deleted file mode 100644 index 1a0eed2a..00000000 --- a/tests/modules/sratools/prefetch/nextflow_mount.config +++ /dev/null @@ -1,17 +0,0 @@ -params.settings_path = '/tmp/.ncbi' -params.settings_file = "${params.settings_path}/user-settings.mkfg" - -env.NCBI_SETTINGS = params.settings_file - -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: SRATOOLS_PREFETCH { - containerOptions = { - (workflow.containerEngine == 'singularity') ? - "-B ${params.settings_path}:${params.settings_path}" : - "-v ${params.settings_path}:${params.settings_path}" - } - } -} diff --git a/tests/modules/sratools/prefetch/test.yml b/tests/modules/sratools/prefetch/test.yml index d747b9a6..a6c213f8 100644 --- a/tests/modules/sratools/prefetch/test.yml +++ b/tests/modules/sratools/prefetch/test.yml @@ -1,17 +1,5 @@ -- name: sratools prefetch test_sratools_prefetch_with_settings_input - command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch_with_settings_input -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow.config - tags: - - sratools/prefetch - - sratools - files: - - path: output/sratools/DRR000774/DRR000774.sra - md5sum: 7647dba20c89c0e3d7ad13842f060eb0 - - path: output/sratools/versions.yml - contains: - - "sratools: 2.11.0" - -- name: sratools prefetch test_sratools_prefetch_without_settings_input - command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch_without_settings_input -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow_mount.config +- name: sratools prefetch test_sratools_prefetch + command: nextflow run ./tests/modules/sratools/prefetch -entry test_sratools_prefetch -c ./tests/config/nextflow.config -c ./tests/modules/sratools/prefetch/nextflow.config tags: - sratools/prefetch - sratools