Merge pull request #400 from SPPearce/crukmi

Cancer Research UK Manchester Institute config

.github/workflows/main.yml

@@ -50,6 +50,7 @@ jobs:
           - "cheaha"
           - "computerome"
           - "crick"
+          - "crukmi"
           - "denbi_qbic"
           - "ebc"
           - "eddie"

README.md

@@ -106,6 +106,7 @@ Currently documentation is available for the following systems:
 - [CHEAHA](docs/cheaha.md)
 - [Computerome](docs/computerome.md)
 - [CRICK](docs/crick.md)
+- [Cancer Research UK Manchester Institute](docs/crukmi.md)
 - [CZBIOHUB_AWS](docs/czbiohub.md)
 - [DENBI_QBIC](docs/denbi_qbic.md)
 - [EBC](docs/ebc.md)
@@ -206,6 +207,7 @@ Currently documentation is available for the following pipelines within specific
 - rnavar
   - [MUNIN](docs/pipeline/rnavar/munin.md)
 - sarek
+  - [Cancer Research UK Manchester Institute](docs/pipeline/sarek/crukmi.md)
   - [MUNIN](docs/pipeline/sarek/munin.md)
   - [UPPMAX](docs/pipeline/sarek/uppmax.md)
 - taxprofiler

conf/crukmi.config

@@ -0,0 +1,52 @@
+//Profile config names for nf-core/configs
+params {
+  config_profile_description = 'Cancer Research UK Manchester Institute HPC cluster profile provided by nf-core/configs'
+  config_profile_contact = 'Stephen Kitcatt, Simon Pearce (@skitcattCRUKMI, @sppearce)'
+  config_profile_url = 'http://scicom.picr.man.ac.uk/projects/user-support/wiki'
+}
+
+env {
+    SINGULARITY_CACHEDIR = '/lmod/nextflow_software'
+}
+
+singularity {
+  enabled = true
+  autoMounts = true
+}
+
+process {
+  beforeScript = 'module load apps/singularity/3.8.0'
+  executor = 'pbs'
+
+  errorStrategy = {task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish'}
+  maxErrors = '-1'
+  maxRetries = 3
+
+  withLabel:process_low {
+    cpus = { check_max( 1 * task.attempt, 'cpus' ) }
+    memory = { check_max( 5.GB * task.attempt, 'memory' ) }
+  }
+
+  withLabel:process_medium {
+    cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+    memory = { check_max( 20.GB * task.attempt, 'memory' ) }
+  }
+
+  withLabel:process_high {
+    cpus = { check_max( 16 * task.attempt, 'cpus' ) }
+    memory = { check_max( 80.GB * task.attempt, 'memory' ) }
+  }
+
+}
+
+executor {
+    name = 'pbs'
+    queueSize = 1000
+    pollInterval = '10 sec'
+}
+
+params {
+  max_memory = 2000.GB
+  max_cpus = 32
+  max_time = 72.h
+}

conf/pipeline/sarek/crukmi.config

@@ -0,0 +1,18 @@
+// Profile config names for nf-core/configs
+
+params {
+  // Specific nf-core/configs params
+  config_profile_description = 'Cancer Research UK Manchester Institute HPC cluster profile provided by nf-core/configs'
+  config_profile_contact = 'Stephen Kitcatt, Simon Pearce (@skitcattCRUKMI, @sppearce)'
+  config_profile_url = 'http://scicom.picr.man.ac.uk/projects/user-support/wiki'
+}
+
+// Specific nf-core/sarek process configuration
+process {
+
+  withName: 'SAMTOOLS_MPILEUP' {
+    cpus   = 1
+    memory = { 5.GB * task.attempt }
+  }
+
+}

docs/crukmi.md

@@ -0,0 +1,15 @@
+# nf-core/configs: Cancer Research UK Manchester Institute Configuration
+
+All nf-core pipelines have been successfully configured for the use on the HPC (phoenix) at Cancer Research UK Manchester Institute.
+
+To use, run the pipeline with `-profile crukmi`. This will download and launch the [`crukmi.config`](../conf/crukmi.config) which has been pre-configured with a setup suitable for the phoenix HPC. Using this profile, singularity images will be downloaded to run on the cluster.
+
+Before running the pipeline you will need to load Nextflow using the environment module system, for example via:
+
+```bash
+## Load Nextflow and Singularity environment modules
+module purge
+module load apps/nextflow/22.04.5
+```
+
+The pipeline should always be executed inside a workspace on the `/scratch/` system. All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory.

docs/pipeline/sarek/crukmi.md

@@ -0,0 +1,17 @@
+# nf-core/configs: CRUK-MI sarek specific configuration
+
+Extra specific configuration for sarek pipeline
+
+## Usage
+
+To use, run the pipeline with `-profile crukmi`.
+
+This will download and launch the sarek specific [`crukmi.config`](../../../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the Cancer Research UK Manchester Institute cluster (phoenix).
+
+Example: `nextflow run nf-core/sarek -profile crukmi`
+
+## Sarek specific configurations for CRUK-MI
+
+Specific configurations for `CRUK-MI` has been made for sarek.
+
+- Initial requested resources for SAMTOOLS_MPILEUP are only 5GB and 1 core.

nfcore_custom.config

@@ -31,6 +31,7 @@ profiles {
   cheaha       { includeConfig "${params.custom_config_base}/conf/cheaha.config" }
   computerome  { includeConfig "${params.custom_config_base}/conf/computerome.config" }
   crick        { includeConfig "${params.custom_config_base}/conf/crick.config" }
+  crukmi       { includeConfig "${params.custom_config_base}/conf/crukmi.config" }
   czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" }
   denbi_qbic   { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" }
   ebc          { includeConfig "${params.custom_config_base}/conf/ebc.config" }

pipeline/sarek.config

@@ -9,10 +9,11 @@
  */
 
 profiles {
-  munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" }
-  uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" }
-  icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" }
   cfc { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" }
   cfc_dev { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" }
+  crukmi { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/crukmi.config" }
   eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" }
+  icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" }
+  munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" }
+  uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" }
 }