Merge pull request #1630 from nf-core/add-ncbi-settings

feat: add module that verifies NCBI settings

modules/custom/sratoolsncbisettings/main.nf

@@ -0,0 +1,20 @@
+process CUSTOM_SRATOOLSNCBISETTINGS {
+    tag 'ncbi-settings'
+    label 'process_low'
+
+    conda (params.enable_conda ? 'bioconda::sra-tools=2.11.0' : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5321ha49a11a_3' :
+        'quay.io/biocontainers/sra-tools:2.11.0--pl5321ha49a11a_3' }"
+
+    output:
+    path('*.mkfg')     , emit: ncbi_settings
+    path 'versions.yml', emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    shell:
+    config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
+    template 'detect_ncbi_settings.sh'
+}

modules/custom/sratoolsncbisettings/meta.yml

@@ -0,0 +1,28 @@
+name: "sratoolsncbisettings"
+description: Test for the presence of suitable NCBI settings or create them on the fly.
+keywords:
+  - NCBI
+  - settings
+  - sra-tools
+  - prefetch
+  - fasterq-dump
+tools:
+  - "sratools":
+      description: "SRA Toolkit and SDK from NCBI"
+      homepage: https://github.com/ncbi/sra-tools
+      documentation: https://github.com/ncbi/sra-tools/wiki
+      tool_dev_url: https://github.com/ncbi/sra-tools
+      licence: "['Public Domain']"
+
+output:
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - ncbi_settings:
+      type: file
+      description: An NCBI user settings file.
+      pattern: "*.mkfg"
+
+authors:
+  - "@Midnighter"

modules/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -u
+
+
+# Get the expected NCBI settings path and define the environment variable
+# `NCBI_SETTINGS`.
+eval "$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+
+# If the user settings do not exist yet, create a file suitable for `prefetch`
+# and `fasterq-dump`. If an existing settings file does not contain the required
+# values, error out with a helpful message.
+if [[ ! -f "${NCBI_SETTINGS}" ]]; then
+    printf '!{config}' > 'user-settings.mkfg'
+else
+    prefetch --help &> /dev/null
+    if [[ $? = 78 ]]; then
+        echo "You have an existing vdb-config at '${NCBI_SETTINGS}' but it is"\
+        "missing the required entries for /LIBS/GUID and"\
+        "/libs/cloud/report_instance_identity."\
+        "Feel free to add the following to your settings file:" >&2
+        echo "$(printf '!{config}')" >&2
+        exit 1
+    fi
+    fasterq-dump --help &> /dev/null
+    if [[ $? = 78 ]]; then
+        echo "You have an existing vdb-config at '${NCBI_SETTINGS}' but it is"\
+        "missing the required entries for /LIBS/GUID and"\
+        "/libs/cloud/report_instance_identity."\
+        "Feel free to add the following to your settings file:" >&2
+        echo "$(printf '!{config}')" >&2
+        exit 1
+    fi
+    if [[ "${NCBI_SETTINGS}" != *.mkfg ]]; then
+        echo "The detected settings '${NCBI_SETTINGS}' do not have the required"\
+        "file extension '.mkfg'." >&2
+        exit 1
+    fi
+    cp "${NCBI_SETTINGS}" ./
+fi
+
+cat <<-END_VERSIONS > versions.yml
+"!{task.process}":
+    sratools: $(vdb-config --version 2>&1 | grep -Eo '[0-9.]+')
+END_VERSIONS

tests/config/pytest_modules.yml

@@ -495,6 +495,10 @@ custom/getchromsizes:
   - modules/custom/getchromsizes/**
   - tests/modules/custom/getchromsizes/**
 
+custom/sratoolsncbisettings:
+  - modules/custom/sratoolsncbisettings/**
+  - tests/modules/custom/sratoolsncbisettings/**
+
 cutadapt:
   - modules/cutadapt/**
   - tests/modules/cutadapt/**

tests/modules/custom/sratoolsncbisettings/main.nf

@@ -0,0 +1,44 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { CUSTOM_SRATOOLSNCBISETTINGS } from '../../../../modules/custom/sratoolsncbisettings/main.nf'
+
+workflow test_sratoolsncbisettings_with_good_existing {
+
+    file(params.settings_path).mkdirs()
+    def settings = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true)
+    settings.copyTo(params.settings_file)
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}
+
+workflow test_sratoolsncbisettings_with_bad_existing {
+
+    file(params.settings_path).mkdirs()
+    def settings = file(params.settings_file)
+    settings.text = '''
+    ## auto-generated configuration file - DO NOT EDIT ##
+
+    config/default = "false"
+    /repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
+    /repository/remote/protected/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
+    /repository/user/ad/public/apps/file/volumes/flatAd = "."
+    /repository/user/ad/public/apps/refseq/volumes/refseqAd = "."
+    /repository/user/ad/public/apps/sra/volumes/sraAd = "."
+    /repository/user/ad/public/apps/sraPileup/volumes/ad = "."
+    /repository/user/ad/public/apps/sraRealign/volumes/ad = "."
+    /repository/user/ad/public/apps/wgs/volumes/wgsAd = "."
+    /repository/user/ad/public/root = "."
+    /repository/user/default-path = "/root/ncbi"
+    '''.stripIndent()
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}
+
+workflow test_sratoolsncbisettings_with_nonexisting {
+    def settings = file(params.settings_file)
+    settings.delete()
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}

tests/modules/custom/sratoolsncbisettings/nextflow.config

@@ -0,0 +1,8 @@
+params.settings_path = '/tmp/.ncbi'
+params.settings_file = "${params.settings_path}/user-settings.mkfg"
+
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+}

tests/modules/custom/sratoolsncbisettings/nextflow_mount.config

@@ -0,0 +1,17 @@
+params.settings_path = '/tmp/.ncbi'
+params.settings_file = "${params.settings_path}/user-settings.mkfg"
+
+env.NCBI_SETTINGS = params.settings_file
+
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: CUSTOM_SRATOOLSNCBISETTINGS {
+        containerOptions = {
+            (workflow.containerEngine == 'singularity') ?
+            "-B ${params.settings_path}:${params.settings_path}" :
+            "-v ${params.settings_path}:${params.settings_path}"
+        }
+    }
+}

tests/modules/custom/sratoolsncbisettings/test.yml

@@ -0,0 +1,44 @@
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_good_existing"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_good_existing -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow_mount.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      md5sum: 955e27aff2c277c2f1f0943a098888c1
+    - path: output/custom/versions.yml
+      contains:
+        - "sratools: 2.11.0"
+
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_bad_existing"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_bad_existing -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow_mount.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  exit_code: 1
+  stdout:
+    contains:
+      - "Command error:"
+      - "missing the required entries"
+      - "/LIBS/GUID"
+      - "/libs/cloud/report_instance_identity"
+      - "Feel free to add the following"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      should_exist: false
+    - path: output/custom/versions.yml
+      should_exist: false
+
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_nonexisting"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_nonexisting -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      contains:
+        - "/LIBS/GUID"
+        - "/libs/cloud/report_instance_identity"
+    - path: output/custom/versions.yml
+      contains:
+        - "sratools: 2.11.0"