Proper user tests now :-)

.github/workflows/main.yml

@@ -19,5 +19,8 @@ jobs:
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
+      - name: Check whether profiles are all tested appropriately
+        run: | 
+          python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml
       - name: "Check profile"
         run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }}

bin/cchecker.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+#######################################################################
+#######################################################################
+## Created on November 26 to check pipeline configs for nf-core/configs
+#######################################################################
+#######################################################################
+
+import os
+import sys
+import argparse
+import re
+
+############################################
+############################################
+## PARSE ARGUMENTS
+############################################
+############################################
+
+Description = 'Double check custom config file and github actions file to test all cases'
+Epilog = """Example usage: python cchecker.py <nfcore_custom.config> <github_actions_file>"""
+
+argParser = argparse.ArgumentParser(description=Description, epilog=Epilog)
+## REQUIRED PARAMETERS
+argParser.add_argument('CUSTOM_CONFIG', help="Input nfcore_custom.config.")
+argParser.add_argument('GITHUB_CONFIG', help="Input Github Actions YAML")
+
+args = argParser.parse_args()
+
+############################################
+############################################
+## MAIN FUNCTION
+############################################
+############################################
+
+def check_config(Config, Github):
+
+    regex = 'includeConfig*'
+    ERROR_STR = 'ERROR: Please check config file! Did you really update the profiles?'
+
+    ## CHECK Config First
+    config_profiles = set()
+    with open(Config, 'r') as cfg:
+        for line in cfg:
+            if re.search(regex, line):
+                hit = line.split('/')[2].split('.')[0]
+                config_profiles.add(hit)
+    
+    ###Check Github Config now
+    tests = set()
+    with open(Github, 'r') as ghfile:
+        for line in ghfile:
+            if re.search('profile: ', line):
+                line = line.replace('\'','').replace('[','').replace(']','')
+                profiles = line.split(':')[1].split(',')
+                for p in profiles:
+                    tests.add(p)
+
+    print(len(tests))
+    print(len(config_profiles))
+    
+    ###Check if sets are equal
+    if tests == config_profiles:
+        sys.exit(0)
+    else: 
+        #Maybe report what is missing here too
+        #print("Tests don't seem to test these profiles properly!\n")
+       # print(config_profiles.difference(tests))
+        sys.exit(1)
+
+check_config(Config=args.CUSTOM_CONFIG,Github=args.GITHUB_CONFIG)

nfcore_custom.config

@@ -11,6 +11,7 @@
 params.custom_config_version = 'master'
 params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
 
+//Please use a new line per includeConfig section to allow easier linting/parsing. Thank you.
 profiles {
   awsbatch     { includeConfig "${params.custom_config_base}/conf/awsbatch.config" }
   bigpurple    { includeConfig "${params.custom_config_base}/conf/bigpurple.config" }
@@ -21,7 +22,8 @@ profiles {
   cfc          { includeConfig "${params.custom_config_base}/conf/cfc.config" }
   crick        { includeConfig "${params.custom_config_base}/conf/crick.config" }
   czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" }
-  czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" }
+  czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; 
+                              includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" }
   denbi_qbic   { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" }
   genouest     { includeConfig "${params.custom_config_base}/conf/genouest.config" }
   gis          { includeConfig "${params.custom_config_base}/conf/gis.config" }
@@ -34,7 +36,8 @@ profiles {
   shh          { includeConfig "${params.custom_config_base}/conf/shh.config" }
   uct_hex      { includeConfig "${params.custom_config_base}/conf/uct_hex.config" }
   uppmax       { includeConfig "${params.custom_config_base}/conf/uppmax.config" }
-  uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" }
+  uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; 
+                 includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" }
   uzh          { includeConfig "${params.custom_config_base}/conf/uzh.config" }
 }