Merge pull request #170 from jfy133/master

Make walltimes more sophisticated and add a HOPS profile for nf-core/eager @ SHH

conf/pipeline/eager/shh.config

@@ -5,12 +5,12 @@ params {
   config_profile_contact = 'James Fellows Yates (@jfy133)'
   config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs'
   igenomes_base = "/projects1/public_data/igenomes/"
- }
+}
 
 // Specific nf-core/eager process configuration
 process {
 
-  maxRetries = 5
+  maxRetries = 2
 
   withName: malt {
     maxRetries = 1
@@ -19,9 +19,90 @@ process {
     time = 1440.h
     queue = { task.memory > 756.GB ? 'supercruncher' : 'long' }
   }
+  
+  withLabel:'sc_tiny'{
+    cpus = { check_max( 1, 'cpus' ) }
+    memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+    time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'sc_small'{
+    cpus = { check_max( 1, 'cpus' ) }
+    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+    time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'sc_medium'{
+    cpus = { check_max( 1, 'cpus' ) }
+    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+    time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'mc_small'{
+    cpus = { check_max( 2, 'cpus' ) }
+    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+    time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'mc_medium' {
+    cpus = { check_max( 4, 'cpus' ) }
+    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+    time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'mc_large'{
+     cpus = { check_max( 8, 'cpus' ) }
+     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+     time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
+  }
+
+  withLabel:'mc_huge'{
+     cpus = { check_max( 32, 'cpus' ) }
+     memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+     time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 2.h }
+   }
+  
 }
 
 profiles {
+  // IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY.
+  hops {
+    params {
+      config_profile_description = 'Rough HOPS screening MPI-SHH profile, provided by nf-core/configs.'
+      fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta'
+      bwa_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta'
+      fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai'
+      seq_dict = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.dict'
+      bwaalnn = 0.01
+      bwaalnl = 16
+      run_bam_filtering = true
+      bam_discard_unmapped = true
+      bam_unmapped_type = 'fastq'
+      run_metagenomic_screening = true
+      metagenomic_tool = 'malt'
+      metagenomic_min_support_reads = 1
+      database = '/projects1/malt/databases/indexed/index040/full-bac-full-vir-etal-nov_2017'
+      percent_identity = 85
+      malt_mode = 'BlastN'
+      malt_alignment_mode = 'SemiGlobal'
+      malt_top_percent = 1
+      malt_min_support_mode = 'reads'
+      malt_max_queries = 100
+      malt_memory_mode = 'load'
+      run_maltextract = true
+      maltextract_taxon_list = '/projects1/users/key/anc5h/soi.backup/List_of_pathogens_KB_fmk12_wViruses1.txt'
+      maltextract_ncbifiles = '/projects1/clusterhomes/huebler/RMASifter/RMA_Extractor_Resources/'
+      maltextract_filter = 'def_anc'
+      maltextract_toppercent = 0.01
+      maltextract_destackingoff = false
+      maltextract_downsamplingoff = false
+      maltextract_duplicateremovaloff = false
+      maltextract_matches = false
+      maltextract_megansummary = true
+      maltextract_percentidentity = 85.0
+      maltextract_topalignment = false
+    }
+  }
   pathogen_loose {
     params {
       config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.'

docs/pipeline/eager/shh.md

@@ -14,5 +14,19 @@ Example: `nextflow run nf-core/eager -profile shh`
 
 Specific configurations for shh has been made for eager.
 
+### General profiles
+
 * If running with the MALT module turned on, the MALT process by default will be sent  to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG.
-* Provides additional group specific profiles, which adapt the `bwa aln` mapping parameters to each context: `pathogens_loose` (`-l 0.01 -n 16`), `pathogens_strict` (`-l 32, -n 0.1`) and `human` (`-l 16500, -n 0.01`).
+
+### Contextual profiles
+
+#### Human Pop-Gen
+
+* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`)
+
+#### Pathogen
+
+* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`)
+* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`)
+* `hops`: profile with default paths and parameters for automated/initial pathogen screening.
+  * :warning: This is NOT a reproducible profile as it contains hardcoded paths. This should only be used for initial/automated screening where you wish to quickly check for any possible positives; after which you should re-do screening in a reproducible manner for publication!