From 2afb4f0ad6674546b320baeb52f360733f6c69f4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 13 Apr 2021 14:37:54 +0200 Subject: [PATCH 1/9] Don't recommend setting --custom_config_base manually --- README.md | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7637725..613ee05 100644 --- a/README.md +++ b/README.md @@ -39,27 +39,22 @@ You should be able to get a good idea as to how other people are customising the ### Offline usage -If you want to use an existing config available in `nf-core/configs`, and you're running on a system that has no internet connection, you'll need to download the config file and place it in a location that is visible to the file system on which you are running the pipeline. -Then run the pipeline with `--custom_config_base` or `params.custom_config_base` set to the location of the directory containing the repository files: +To use nf-core pipelines offline, we recommend using the `nf-core download` helper tool. This will download both the pipeline files and also the config profiles from `nf-core/configs`. The pipeline files are then edited to load the configs from their relative file path correctly. ```bash -## Download and unzip the config files -cd /path/to/my/configs -wget https://github.com/nf-core/configs/archive/master.zip -unzip master.zip - -## Run the pipeline -cd /path/to/my/data -nextflow run /path/to/pipeline/ --custom_config_base /path/to/my/configs/configs-master/ +# Download the workflow + transfer to offline cluster +nf-core download rnaseq +scp nf-core-rnaseq-3.0.tar.gz me@myserver.com:/path/to/workflows # or however you prefer to transfer files to your offline cluster +ssh me@myserver.com +# Extract workflow files +cd /path/to/workflows +tar -xzf nf-core-rnaseq-3.0.tar.gz +# Run workflow +cd /path/to/data +nextflow run /path/to/workflows/nf-core-rnaseq-3.0/workflow -profile mycluster ``` -Alternatively, instead of using the configuration profiles from this repository, you can run your pipeline directly calling the single institutional config file that you need with the `-c` parameter. - -```bash -nextflow run /path/to/pipeline/ -c /path/to/my/configs/configs-master/conf/my_config.config -``` - -> Note that the nf-core/tools helper package has a `download` command to download all required pipeline files + singularity containers + institutional configs in one go for you, to make this process easier. +If you prefer, you can download these config profiles yourself and customise the `--custom_config_base` / `params.custom_config_base` parameter in each pipeline to to set to the location of the configs directory. ## Adding a new config From 2f343868adf18aed131194082230fd84ac210fef Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 13 Apr 2021 22:52:31 +0200 Subject: [PATCH 2/9] Apply suggestions from code review Co-authored-by: Maxime Garcia --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 613ee05..4ccc10a 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ To use nf-core pipelines offline, we recommend using the `nf-core download` help # Download the workflow + transfer to offline cluster nf-core download rnaseq scp nf-core-rnaseq-3.0.tar.gz me@myserver.com:/path/to/workflows # or however you prefer to transfer files to your offline cluster +# Connect to offline cluster ssh me@myserver.com # Extract workflow files cd /path/to/workflows From c7e4ea25f5b2618d3ba6e7efd18146ba45d25113 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 14 Apr 2021 23:30:17 +0200 Subject: [PATCH 3/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ccc10a..5feb3cc 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cd /path/to/data nextflow run /path/to/workflows/nf-core-rnaseq-3.0/workflow -profile mycluster ``` -If you prefer, you can download these config profiles yourself and customise the `--custom_config_base` / `params.custom_config_base` parameter in each pipeline to to set to the location of the configs directory. +If required, you can instead download the nf-core/configs files yourself and customise the `--custom_config_base` / `params.custom_config_base` parameter in each pipeline to to set to the location of the configs directory. ## Adding a new config From 00491de038a81906e11339d7b783bc74c59597af Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 3 Aug 2021 13:54:14 +0200 Subject: [PATCH 4/9] Update eva.config --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 87d679e..359ca8b 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -69,7 +69,7 @@ process { } withName: adapter_removal { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 4)}G,virtual_free=${(task.memory.toGiga() * 4)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 5)}G,virtual_free=${(task.memory.toGiga() * 5)}G" } } withName: dedup { From 76fc461670252883ac61cc099458178ce602d774 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 4 Aug 2021 10:14:56 +0200 Subject: [PATCH 5/9] Try new strategy for SGE/JAVA bs --- conf/pipeline/eager/eva.config | 236 +++++++++++++++++++++++++++++++-- 1 file changed, 224 insertions(+), 12 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 359ca8b..197e930 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -69,7 +69,7 @@ process { } withName: adapter_removal { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 5)}G,virtual_free=${(task.memory.toGiga() * 5)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: dedup { @@ -77,11 +77,11 @@ process { } withName: markduplicates { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: library_merge { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: malt { @@ -97,7 +97,7 @@ process { } withName: mtnucratio { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 4)}G,virtual_free=${(task.memory.toGiga() * 4)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: vcf2genome { @@ -105,11 +105,11 @@ process { } withName: qualimap { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: damageprofiler { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: circularmapper { @@ -129,6 +129,145 @@ process { profiles { + medium_data { + + params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager medium-data EVA profile provided by nf-core/configs' + } + + executor { + queueSize = 8 + } + + process { + + beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"' + + maxRetries = 2 + + // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion + clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.5.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 24.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use + + withName: makeSeqDict { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: fastqc { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: adapter_removal { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: dedup { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: markduplicates { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: library_merge { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: malt { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName:hostremoval_input_fastq { + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = 1440.h + } + + withName: maltextract { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: multivcfanalyzer { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: mtnucratio { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: vcf2genome { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: qualimap { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: damageprofiler { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } + + withName: circularmapper { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + } + + withName: circulargenerator { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + } + + withName: preseq { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' } + } + + } + } + big_data { params { @@ -143,13 +282,12 @@ profiles { process { - maxRetries = 2 + beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"' - withName:hostremoval_input_fastq { - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = 1440.h - } + maxRetries = 2 + + // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion + clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" } withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } @@ -192,6 +330,80 @@ profiles { memory = { check_max( 512.GB * task.attempt, 'memory' ) } time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } + + // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use + + withName: makeSeqDict { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: fastqc { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: adapter_removal { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: dedup { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: markduplicates { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: library_merge { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName:hostremoval_input_fastq { + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = 1440.h + } + + withName: malt { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: maltextract { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: multivcfanalyzer { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: mtnucratio { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: vcf2genome { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: qualimap { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: damageprofiler { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: circularmapper { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: circulargenerator { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + + withName: preseq { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' } + } + } } From b0e344846d68117aff7d06037db40813b1608a15 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 4 Aug 2021 10:37:49 +0200 Subject: [PATCH 6/9] Changes after code review from @aidaanva --- conf/pipeline/eager/eva.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 197e930..16e185a 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -253,11 +253,11 @@ profiles { } withName: circularmapper { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } } withName: circulargenerator { - clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } } withName: preseq { From 670135406080fed543aa42d2eb428305dde02042 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 20 Aug 2021 10:11:11 +0200 Subject: [PATCH 7/9] Add genotyping_ug to SGE fixes --- conf/pipeline/eager/eva.config | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 16e185a..880df27 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -124,6 +124,10 @@ process { clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' } } + + withName: genotyping_ug { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + } } @@ -259,6 +263,10 @@ profiles { withName: circulargenerator { clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } } + + withName: genotyping_ug { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + } withName: preseq { clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } @@ -398,7 +406,11 @@ profiles { withName: circulargenerator { clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } } - + + withName: genotyping_ug { + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + } + withName: preseq { clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' } From 59af7f0eee7effac0e2847b346e6b6af5ca28a96 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 20 Aug 2021 10:15:39 +0200 Subject: [PATCH 8/9] Delete SHH profiles due to decommissioning --- README.md | 2 - conf/pipeline/eager/shh.config | 220 --------------------------------- conf/shh.config | 35 ------ docs/pipeline/eager/shh.md | 36 ------ docs/shh.md | 16 --- nextflow.config | 2 +- nfcore_custom.config | 1 - pipeline/eager.config | 1 - 8 files changed, 1 insertion(+), 312 deletions(-) delete mode 100644 conf/pipeline/eager/shh.config delete mode 100644 conf/shh.config delete mode 100644 docs/pipeline/eager/shh.md delete mode 100644 docs/shh.md diff --git a/README.md b/README.md index 29fbce2..0730271 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,6 @@ Currently documentation is available for the following systems: * [PRINCE](docs/prince.md) * [SANGER](docs/sanger.md) * [SEG_GLOBE](docs/seg_globe.md) -* [SHH](docs/shh.md) * [UCT_HPC](docs/uct_hpc.md) * [UPPMAX](docs/uppmax.md) * [UTD_GANYMEDE](docs/utd_ganymede.md) @@ -180,7 +179,6 @@ Currently documentation is available for the following pipelines within specific * [BINAC](docs/pipeline/ampliseq/binac.md) * [UPPMAX](docs/pipeline/ampliseq/uppmax.md) * eager - * [SHH](docs/pipeline/eager/shh.md) * [EVA](docs/pipeline/eager/eva.md) * rnafusion * [MUNIN](docs/pipeline/rnafusion/munin.md) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config deleted file mode 100644 index b99d038..0000000 --- a/conf/pipeline/eager/shh.config +++ /dev/null @@ -1,220 +0,0 @@ -// Profile config names for nf-core/configs - -params { - // Specific nf-core/configs params - config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' - igenomes_base = "/projects1/public_data/igenomes/" -} - -// Specific nf-core/eager process configuration -process { - - maxRetries = 3 - - withName: malt { - maxRetries = 1 - memory = { task.attempt > 1 ? 1900.GB : 725.GB } - cpus = { task.attempt > 1 ? 112 : 64 } - time = 1440.h - queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } - } - - withName: circulargenerator { - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } - } - - withLabel:'sc_tiny'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'sc_small'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'sc_medium'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'mc_small'{ - cpus = { check_max( 2, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'mc_medium' { - cpus = { check_max( 4, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - - withLabel:'mc_huge'{ - cpus = { check_max( 32, 'cpus' ) } - memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } - } - -} - -profiles { - - big_data { - - params { - // Specific nf-core/configs params - config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager big-data SHH profile provided by nf-core/configs' - } - - executor { - queueSize = 6 - } - - process { - - maxRetries = 2 - - withName:malt { - maxRetries = 1 - memory = { task.attempt > 1 ? 1900.GB : 725.GB } - cpus = { task.attempt > 1 ? 112 : 64 } - time = 1440.h - queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } - } - - withName:hostremoval_input_fastq { - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = 1440.h - queue = 'long' - } - - withLabel:'sc_tiny'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 2.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'sc_small'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'sc_medium'{ - cpus = { check_max( 1, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'mc_small'{ - cpus = { check_max( 2, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'mc_medium' { - cpus = { check_max( 4, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } - memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - - withLabel:'mc_huge'{ - cpus = { check_max( 32, 'cpus' ) } - memory = { check_max( 512.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : 48.h } - } - } - } - - microbiome_screening { - process { - // Assuming NCBI NT-sized DB - withName: malt { - maxRetries = 1 - memory = 1900.GB - cpus = 112 - time = 1440.h - queue = 'supercruncher' - } - } - } - // IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY. - hops { - params { - config_profile_description = 'Rough HOPS screening MPI-SHH profile, provided by nf-core/configs.' - fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' - bwa_index = '/projects1/Reference_Genomes/Human/HG19/' - fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai' - seq_dict = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.dict' - bwaalnn = 0.01 - bwaalnl = 16 - run_bam_filtering = true - bam_unmapped_type = 'fastq' - run_metagenomic_screening = true - metagenomic_tool = 'malt' - metagenomic_min_support_reads = 1 - database = '/projects1/malt/databases/indexed/index040/full-bac-full-vir-etal-nov_2017' - percent_identity = 85 - malt_mode = 'BlastN' - malt_alignment_mode = 'SemiGlobal' - malt_top_percent = 1 - malt_min_support_mode = 'reads' - malt_max_queries = 100 - malt_memory_mode = 'load' - run_maltextract = true - maltextract_taxon_list = '/projects1/users/key/anc5h/soi.backup/List_of_pathogens_KB_fmk12_wViruses1.txt' - maltextract_ncbifiles = '/projects1/clusterhomes/huebler/RMASifter/RMA_Extractor_Resources/' - maltextract_filter = 'def_anc' - maltextract_toppercent = 0.01 - maltextract_destackingoff = false - maltextract_downsamplingoff = false - maltextract_duplicateremovaloff = false - maltextract_matches = false - maltextract_megansummary = true - maltextract_percentidentity = 85.0 - maltextract_topalignment = false - } - } - pathogen_loose { - params { - config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.' - bwaalnn = 0.01 - bwaalnl = 16 - } - } - pathogen_strict { - params { - config_profile_description = 'Pathogen (strict) MPI-SHH SDAG profile, provided by nf-core/configs.' - bwaalnn = 0.1 - bwaalnl = 32 - } - } - human { - params { - config_profile_description = 'Human MPI-SHH SDAG profile, provided by nf-core/configs.' - bwaalnn = 0.01 - bwaalnl = 16500 - } - } -} diff --git a/conf/shh.config b/conf/shh.config deleted file mode 100644 index a17c935..0000000 --- a/conf/shh.config +++ /dev/null @@ -1,35 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'Generic MPI-SHH SDAG cluster profile provided by nf-core/configs.' - config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' - config_profile_url = 'https://shh.mpg.de' - max_memory = 2.TB - max_cpus = 128 - max_time = 720.h - //Illumina iGenomes reference file path - igenomes_base = "/projects1/public_data/igenomes/" -} - -// Preform work directory cleanup after a successful run -cleanup = true - -process { - executor = 'slurm' - queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } -} - -executor { - queueSize = 8 -} - -profiles { - singularity { - singularity.enabled = true - singularity.autoMounts = true - singularity.cacheDir = "/projects1/singularity_scratch/cache/" - } - // Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option. - debug { - cleanup = false - } -} diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md deleted file mode 100644 index c09606d..0000000 --- a/docs/pipeline/eager/shh.md +++ /dev/null @@ -1,36 +0,0 @@ -# nf-core/configs: shh eager specific configuration - -Extra specific configuration for eager pipeline - -## Usage - -To use, run the pipeline with `-profile shh`. - -This will download and launch the eager specific [`shh.config`](../../../conf/pipeline/eager/shh.config) which has been pre-configured with a setup suitable for the shh cluster. - -Example: `nextflow run nf-core/eager -profile shh` - -## eager specific configurations for shh - -Specific configurations for shh has been made for eager. - -### General profiles - -* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG. - -### Contextual profiles - -#### Microbiome Sciences - -* `microbiome_screening` runs MALT straight to supercruncher (with no retries!) and full resources requested due to microbiome screening databases often easily reach this size - -#### Human Pop-Gen - -* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) - -#### Pathogen - -* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) -* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) -* `hops`: profile with default paths and parameters for automated/initial pathogen screening. - * :warning: This is NOT a reproducible profile as it contains hardcoded paths. This should only be used for initial/automated screening where you wish to quickly check for any possible positives; after which you should re-do screening in a reproducible manner for publication! diff --git a/docs/shh.md b/docs/shh.md deleted file mode 100644 index 6f61426..0000000 --- a/docs/shh.md +++ /dev/null @@ -1,16 +0,0 @@ -# nf-core/configs: SHH Configuration - -All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de). - -To use, run the pipeline with `-profile shh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile shh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: - -```bash -/projects1/singularity_scratch/cache/ -``` - -however this will likely change to a read-only directory in the future that will be managed by the IT team. - -This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile shh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. - ->NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team. ->NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. diff --git a/nextflow.config b/nextflow.config index f1ffe8a..fab2cd1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1 +1 @@ -includeConfig("nfcore_custom.config") \ No newline at end of file +includeConfig("nfcore_custom.config") diff --git a/nfcore_custom.config b/nfcore_custom.config index 74d25b9..99a31e4 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -48,7 +48,6 @@ profiles { prince { includeConfig "${params.custom_config_base}/conf/prince.config" } sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} - shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" } diff --git a/pipeline/eager.config b/pipeline/eager.config index 2827496..3e47276 100644 --- a/pipeline/eager.config +++ b/pipeline/eager.config @@ -9,7 +9,6 @@ */ profiles { - shh { includeConfig "${params.custom_config_base}/conf/pipeline/eager/shh.config" } mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } eva { includeConfig "${params.custom_config_base}/conf/pipeline/eager/eva.config" } } From f41cb5986f35a230469fd4b2c104559a1d3b92cb Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 20 Aug 2021 10:18:04 +0200 Subject: [PATCH 9/9] remove from testing --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5e6a57..12d23f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,6 @@ jobs: - 'prince' - 'sanger' - 'seg_globe' - - 'shh' - 'uct_hpc' - 'uppmax' - 'utd_ganymede'