1
0
Fork 0
mirror of https://github.com/MillironX/nf-configs.git synced 2024-11-22 00:26:03 +00:00

Merge pull request #256 from ggabernet/master

Add AWS tower config
This commit is contained in:
Gisela Gabernet 2021-07-03 16:54:59 +02:00 committed by GitHub
commit 435955ab20
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 86 additions and 12 deletions

View file

@ -16,7 +16,49 @@ jobs:
needs: test_all_profiles
strategy:
matrix:
profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'sanger', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'utd_sysbio', 'uzh']
profile:
- 'abims'
- 'awsbatch'
- 'aws_tower'
- 'bi'
- 'bigpurple'
- 'binac'
- 'biohpc_gen'
- 'cbe'
- 'ccga_dx'
- 'ccga_med'
- 'cfc'
- 'cfc_dev'
- 'crick'
- 'denbi_qbic'
- 'ebc'
- 'eddie'
- 'eva'
- 'genotoul'
- 'genouest'
- 'gis'
- 'google'
- 'hebbe'
- 'icr_davros'
- 'ifb_core'
- 'imperial'
- 'imperial_mb'
- 'jax'
- 'kraken'
- 'mpcdf'
- 'munin'
- 'oist'
- 'pasteur'
- 'phoenix'
- 'prince'
- 'sanger'
- 'seg_globe'
- 'shh'
- 'uct_hpc'
- 'uppmax'
- 'utd_ganymede'
- 'utd_sysbio'
- 'uzh'
steps:
- uses: actions/checkout@v1
- name: Install Nextflow

View file

@ -96,6 +96,7 @@ Currently documentation is available for the following systems:
* [ABIMS](docs/abims.md)
* [AWSBATCH](docs/awsbatch.md)
* [AWS_TOWER](docs/aws_tower.md)
* [BIGPURPLE](docs/bigpurple.md)
* [BI](docs/bi.md)
* [BINAC](docs/binac.md)

View file

@ -10,6 +10,7 @@ import os
import sys
import argparse
import re
import yaml
############################################
############################################
@ -51,19 +52,22 @@ def check_config(Config, Github):
### Ignore these profiles
ignore_me = ['czbiohub_aws']
tests.update(ignore_me)
# parse yaml GitHub actions file
try:
with open(Github, 'r') as ghfile:
for line in ghfile:
if re.search('profile: ', line):
line = line.replace('\'','').replace('[','').replace(']','').replace('\n','')
profiles = line.split(':')[1].split(',')
for p in profiles:
tests.add(p.strip())
wf = yaml.safe_load(ghfile)
profile_list = wf["jobs"]["profile_test"]["strategy"]["matrix"]["profile"]
except Exception as e:
print("Could not parse yaml file: {}, {}".format(Github, e))
sys.exit(1)
# Add profiles to test
for profile in profile_list:
tests.add(profile.strip())
###Check if sets are equal
if tests == config_profiles:
sys.exit(0)
else:
#Maybe report what is missing here too
try:
assert tests == config_profiles
except (AssertionError):
print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n")
print(config_profiles.symmetric_difference(tests))
sys.exit(1)

21
conf/aws_tower.config Normal file
View file

@ -0,0 +1,21 @@
//Nextflow config file for running on AWS batch
params {
config_profile_description = 'AWS Batch with Tower Profile'
config_profile_contact = 'Gisela Gabernet (@ggabernet)'
config_profile_url = 'https://aws.amazon.com/batch/'
}
timeline {
overwrite = true
}
report {
overwrite = true
}
trace {
overwrite = true
}
dag {
overwrite = true
}
process.executor = 'awsbatch'

5
docs/aws_tower.md Normal file
View file

@ -0,0 +1,5 @@
# nf-core/configs: AWS Batch with Tower Configuration
To be used when submitting jobs to AWS Batch by using Tower Forge. If you are not using Tower Forge, consider using the profile `awsbatch` where you can directly specify the Batch queue, AWS region and AWS cli path.
This profile defines `awsbatch` as executor, and allows `overwrite` of `trace`, `timeline`, `report` and `dag` to allow resuming pipelines.

View file

@ -12,6 +12,7 @@
profiles {
abims { includeConfig "${params.custom_config_base}/conf/abims.config" }
awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" }
aws_tower { includeConfig "${params.custom_config_base}/conf/aws_tower.config" }
bi { includeConfig "${params.custom_config_base}/conf/bi.config" }
bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" }
binac { includeConfig "${params.custom_config_base}/conf/binac.config" }