Merge pull request #256 from ggabernet/master

Add AWS tower config

.github/workflows/main.yml

@@ -16,7 +16,49 @@ jobs:
     needs: test_all_profiles
     strategy:
         matrix:
-          profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'sanger', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'utd_sysbio', 'uzh']
+          profile: 
+            - 'abims'
+            - 'awsbatch'
+            - 'aws_tower'
+            - 'bi'
+            - 'bigpurple'
+            - 'binac'
+            - 'biohpc_gen'
+            - 'cbe'
+            - 'ccga_dx'
+            - 'ccga_med'
+            - 'cfc'
+            - 'cfc_dev'
+            - 'crick'
+            - 'denbi_qbic'
+            - 'ebc'
+            - 'eddie'
+            - 'eva'
+            - 'genotoul'
+            - 'genouest'
+            - 'gis'
+            - 'google'
+            - 'hebbe'
+            - 'icr_davros'
+            - 'ifb_core'
+            - 'imperial'
+            - 'imperial_mb'
+            - 'jax'
+            - 'kraken'
+            - 'mpcdf'
+            - 'munin'
+            - 'oist'
+            - 'pasteur'
+            - 'phoenix'
+            - 'prince'
+            - 'sanger'
+            - 'seg_globe'
+            - 'shh'
+            - 'uct_hpc'
+            - 'uppmax'
+            - 'utd_ganymede'
+            - 'utd_sysbio'
+            - 'uzh'
     steps:
       - uses: actions/checkout@v1
       - name: Install Nextflow

README.md

@@ -96,6 +96,7 @@ Currently documentation is available for the following systems:
 
 * [ABIMS](docs/abims.md)
 * [AWSBATCH](docs/awsbatch.md)
+* [AWS_TOWER](docs/aws_tower.md)
 * [BIGPURPLE](docs/bigpurple.md)
 * [BI](docs/bi.md)
 * [BINAC](docs/binac.md)

bin/cchecker.py

@@ -10,6 +10,7 @@ import os
 import sys
 import argparse
 import re
+import yaml
 
 ############################################
 ############################################
@@ -51,19 +52,22 @@ def check_config(Config, Github):
     ### Ignore these profiles
     ignore_me = ['czbiohub_aws']
     tests.update(ignore_me)
-    with open(Github, 'r') as ghfile:
+    # parse yaml GitHub actions file
-        for line in ghfile:
+    try:
-            if re.search('profile: ', line):
+        with open(Github, 'r') as ghfile:
-                line = line.replace('\'','').replace('[','').replace(']','').replace('\n','')
+            wf = yaml.safe_load(ghfile)
-                profiles = line.split(':')[1].split(',')
+            profile_list = wf["jobs"]["profile_test"]["strategy"]["matrix"]["profile"]
-                for p in profiles:
+    except Exception as e:
-                    tests.add(p.strip())
+        print("Could not parse yaml file: {}, {}".format(Github, e))
+        sys.exit(1)
+    # Add profiles to test
+    for profile in profile_list:
+        tests.add(profile.strip())
 
     ###Check if sets are equal
-    if tests == config_profiles:
+    try:
-        sys.exit(0)
+        assert tests == config_profiles
-    else:
+    except (AssertionError):
-        #Maybe report what is missing here too
         print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n")
         print(config_profiles.symmetric_difference(tests))
         sys.exit(1)

conf/aws_tower.config

@@ -0,0 +1,21 @@
+//Nextflow config file for running on AWS batch
+params {
+    config_profile_description = 'AWS Batch with Tower Profile'
+    config_profile_contact = 'Gisela Gabernet (@ggabernet)'
+    config_profile_url = 'https://aws.amazon.com/batch/'
+}
+
+timeline {
+    overwrite = true
+}
+report {
+    overwrite = true
+}
+trace {
+    overwrite = true
+}
+dag {
+    overwrite = true
+}
+
+process.executor = 'awsbatch'

docs/aws_tower.md

@@ -0,0 +1,5 @@
+# nf-core/configs: AWS Batch with Tower Configuration
+
+To be used when submitting jobs to AWS Batch by using Tower Forge. If you are not using Tower Forge, consider using the profile `awsbatch` where you can directly specify the Batch queue, AWS region and AWS cli path.
+
+This profile defines `awsbatch` as executor, and allows `overwrite` of `trace`, `timeline`, `report` and `dag` to allow resuming pipelines.

nfcore_custom.config

@@ -12,6 +12,7 @@
 profiles {
   abims        { includeConfig "${params.custom_config_base}/conf/abims.config" }
   awsbatch     { includeConfig "${params.custom_config_base}/conf/awsbatch.config" }
+  aws_tower    { includeConfig "${params.custom_config_base}/conf/aws_tower.config" }
   bi           { includeConfig "${params.custom_config_base}/conf/bi.config" }
   bigpurple    { includeConfig "${params.custom_config_base}/conf/bigpurple.config" }
   binac        { includeConfig "${params.custom_config_base}/conf/binac.config" }