Fix subworkflows seperate from modules (#933)

* ci: Remove pytest_subworkflows

* ci(bam_sort_samtools): Depend on paths-filter instead of pytest-workflow

Co-authored-by: Harshil Patel <drpatelhh@gmail.com>
Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>
Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com>

* ci: Revert back to one job branch

* ci(align_bowtie2): Run tests that depend on bam_sort_samtools

* ci: Fix anchor not being created yet

* ci: Update sra_fastq tags and pytest_modules

* fix(bam_sort_samtools): Update nextflow.config with params

* test(subworkflows): Update gatk_create_som_pon tags

* ci: Point to subworkflow_hacks branch of nf-core tools

Co-authored-by: Harshil Patel <drpatelhh@gmail.com>
Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>
Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com>

.github/workflows/nf-core-linting.yml

@@ -60,7 +60,7 @@ jobs:
 
         # FIXME: Remove this when nf-core modules lint stabilizes and install stable release
       - name: Install nf-core tools development version
-        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@subworkflow_hacks
 
       - name: Install Nextflow
         env:

.github/workflows/pytest-workflow.yml

@@ -6,11 +6,8 @@ on:
     branches: [master]
 
 jobs:
-  ###########
+  changes:
-  # Modules #
+    name: Check for changes
-  ###########
-  module_changes:
-    name: Check for changes in the modules
     runs-on: ubuntu-latest
     outputs:
       # Expose matched filters as job 'modules' output variable
@@ -23,120 +20,17 @@ jobs:
         with:
           filters: "tests/config/pytest_modules.yml"
 
-  module_test:
+  test:
     runs-on: ubuntu-20.04
 
     name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
-    needs: module_changes
+    needs: changes
-    if: needs.module_changes.outputs.modules != '[]'
+    if: needs.changes.outputs.modules != '[]'
     strategy:
       fail-fast: false
       matrix:
         nxf_version: ["21.04.0"]
-        tags: ${{ fromJson(needs.module_changes.outputs.modules) }}
+        tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
-        profile: ["docker", "singularity", "conda"]
-    env:
-      NXF_ANSI_LOG: false
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-
-      - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pytest-workflow
-
-      - uses: actions/cache@v2
-        with:
-          path: /usr/local/bin/nextflow
-          key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }}
-          restore-keys: |
-            ${{ runner.os }}-nextflow-
-
-      - name: Install Nextflow
-        env:
-          NXF_VER: ${{ matrix.nxf_version }}
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-
-      - name: Set up Singularity
-        if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-singularity@v5
-        with:
-          singularity-version: 3.7.1
-
-      - name: Setup miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
-
-      - name: Conda clean
-        if: matrix.profile == 'conda'
-        run: conda clean -a
-
-      # Test the module
-      - name: Run pytest-workflow
-        # only use one thread for pytest-workflow to avoid race condition on conda cache.
-        run: NF_CORE_MODULES_TEST=1 TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof
-
-      - name: Upload logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v2
-        with:
-          name: logs-${{ matrix.profile }}-${{ matrix.nxf_version }}
-          path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/work
-
-  ################
-  # Subworkflows #
-  ################
-  subworkflow_changes:
-    name: Check for changes in the subworkflows
-    runs-on: ubuntu-latest
-    outputs:
-      # Expose matched filters as job 'subworkflows' output variable
-      subworkflows: ${{ steps.filter.outputs.changes }}
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: "tests/config/pytest_subworkflows.yml"
-
-  subworkflow_test:
-    runs-on: ubuntu-20.04
-
-    name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
-    needs: subworkflow_changes
-    if: needs.subworkflow_changes.outputs.subworkflows != '[]'
-    strategy:
-      fail-fast: false
-      matrix:
-        nxf_version: ["21.04.0"]
-        tags: ${{ fromJson(needs.subworkflow_changes.outputs.subworkflows) }}
         profile: ["docker", "singularity", "conda"]
     env:
       NXF_ANSI_LOG: false

subworkflows/nf-core/bam_sort_samtools/nextflow.config

@@ -1 +1,3 @@
-params.options = [:]
+params.sort_options  = [:]
+params.index_options = [:]
+params.stats_options = [:]

tests/config/pytest_modules.yml

@@ -466,7 +466,7 @@ gatk4/createsequencedictionary:
   - modules/gatk4/createsequencedictionary/**
   - tests/modules/gatk4/createsequencedictionary/**
 
-gatk4/createsomaticpanelofnormals:
+gatk4/createsomaticpanelofnormals: &gatk4/createsomaticpanelofnormals
   - modules/gatk4/createsomaticpanelofnormals/**
   - tests/modules/gatk4/createsomaticpanelofnormals/**
 
@@ -482,7 +482,7 @@ gatk4/filtermutectcalls:
   - modules/gatk4/filtermutectcalls/**
   - tests/modules/gatk4/filtermutectcalls/**
 
-gatk4/genomicsdbimport:
+gatk4/genomicsdbimport: &gatk4/genomicsdbimport
   - modules/gatk4/genomicsdbimport/**
   - tests/modules/gatk4/genomicsdbimport/**
 
@@ -1163,7 +1163,7 @@ samtools/idxstats:
   - modules/samtools/idxstats/**
   - tests/modules/samtools/idxstats/**
 
-samtools/index:
+samtools/index: &samtools/index
   - modules/samtools/index/**
   - tests/modules/samtools/index/**
 
@@ -1175,7 +1175,7 @@ samtools/mpileup:
   - modules/samtools/mpileup/**
   - tests/modules/samtools/mpileup/**
 
-samtools/sort:
+samtools/sort: &samtools/sort
   - modules/samtools/sort/**
   - tests/modules/samtools/sort/**
 
@@ -1251,11 +1251,11 @@ spatyper:
   - modules/spatyper/**
   - tests/modules/spatyper/**
 
-sratools/fasterqdump:
+sratools/fasterqdump: &sratools/fasterqdump
   - modules/sratools/fasterqdump/**
   - tests/modules/sratools/fasterqdump/**
 
-sratools/prefetch:
+sratools/prefetch: &sratools/prefetch
   - modules/sratools/prefetch/**
   - tests/modules/sratools/prefetch/**
 
@@ -1374,3 +1374,31 @@ yara/index:
 yara/mapper:
   - modules/yara/mapper/**
   - tests/modules/yara/mapper/**
+
+subworkflows/bam_stats_samtools: &subworkflows/bam_stats_samtools
+  - subworkflows/nf-core/bam_stats_samtools/**
+  - tests/subworkflows/nf-core/bam_stats_samtools/**
+
+subworkflows/bam_sort_samtools: &subworkflows/bam_sort_samtools
+  - subworkflows/nf-core/bam_sort_samtools/**
+  - tests/subworkflows/nf-core/bam_sort_samtools/**
+  - *samtools/sort
+  - *samtools/index
+  - *subworkflows/bam_stats_samtools
+
+subworkflows/align_bowtie2:
+  - subworkflows/nf-core/align_bowtie2/**
+  - tests/subworkflows/nf-core/align_bowtie2/**
+  - *subworkflows/bam_sort_samtools
+
+subworkflows/sra_fastq:
+  - subworkflows/nf-core/sra_fastq/**
+  - tests/subworkflows/nf-core/sra_fastq/**
+  - *sratools/fasterqdump
+  - *sratools/prefetch
+
+subworkflows/gatk_create_som_pon:
+  - subworkflows/nf-core/gatk_create_som_pon/**
+  - tests/subworkflows/nf-core/gatk_create_som_pon/**
+  - *gatk4/genomicsdbimport
+  - *gatk4/createsomaticpanelofnormals

tests/config/pytest_subworkflows.yml

@@ -1,21 +0,0 @@
-subworkflows/align_bowtie2:
-  - subworkflows/nf-core/align_bowtie2/**
-  - tests/subworkflows/nf-core/align_bowtie2/**
-
-subworkflows/bam_stats_samtools:
-  - subworkflows/nf-core/bam_stats_samtools/**
-  - tests/subworkflows/nf-core/bam_stats_samtools/**
-
-subworkflows/bam_sort_samtools:
-  - subworkflows/nf-core/bam_sort_samtools/**
-  - tests/subworkflows/nf-core/bam_sort_samtools/**
-
-subworkflows/sra_fastq:
-  - subworkflows/nf-core/sra_fastq/**
-  - tests/subworkflows/nf-core/sra_fastq/**
-
-subworkflows/gatk_create_som_pon:
-  - subworkflows/nf-core/gatk_create_som_pon/**
-  - tests/subworkflows/nf-core/gatk_create_som_pon/**
-  
-  

tests/modules/sratools/fasterqdump/test.yml

@@ -1,8 +1,7 @@
 - name: sratools fasterqdump test_sratools_fasterqdump_single_end
   command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_single_end -c tests/config/nextflow.config
   tags:
-    - sratools
+    - subworkflows/sra_fastq
-    - sratools/fasterqdump
   files:
     - path: output/sratools/SRR13255544.fastq.gz
       md5sum: 1054c7b71884acdb5eed8a378f18be82
@@ -12,8 +11,7 @@
 - name: sratools fasterqdump test_sratools_fasterqdump_paired_end
   command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c tests/config/nextflow.config
   tags:
-    - sratools
+    - subworkflows/sra_fastq
-    - sratools/fasterqdump
   files:
     - path: output/sratools/SRR11140744_1.fastq.gz
       md5sum: 193809c784a4ea132ab2a253fa4f55b6

tests/subworkflows/nf-core/align_bowtie2/test.yml

@@ -5,14 +5,14 @@
     - subworkflows/bam_sort_samtools
     - subworkflows/bam_stats_samtools
     # Modules
-    - bowtie2
+    # - bowtie2
-    - bowtie2/align
+    # - bowtie2/align
-    - samtools
+    # - samtools
-    - samtools/index
+    # - samtools/index
-    - samtools/sort
+    # - samtools/sort
-    - samtools/stats
+    # - samtools/stats
-    - samtools/idxstats
+    # - samtools/idxstats
-    - samtools/flagstat
+    # - samtools/flagstat
   files:
     - path: ./output/bowtie2/test.bam
     - path: ./output/bowtie2/test.bowtie2.log
@@ -46,14 +46,14 @@
     - subworkflows/bam_sort_samtools
     - subworkflows/bam_stats_samtools
     # Modules
-    - bowtie2
+    # - bowtie2
-    - bowtie2/align
+    # - bowtie2/align
-    - samtools
+    # - samtools
-    - samtools/index
+    # - samtools/index
-    - samtools/sort
+    # - samtools/sort
-    - samtools/stats
+    # - samtools/stats
-    - samtools/idxstats
+    # - samtools/idxstats
-    - samtools/flagstat
+    # - samtools/flagstat
   files:
     - path: ./output/bowtie2/test.bam
     - path: ./output/bowtie2/test.bowtie2.log

tests/subworkflows/nf-core/bam_sort_samtools/test.yml

@@ -4,12 +4,12 @@
     - subworkflows/bam_sort_samtools
     - subworkflows/bam_stats_samtools
     # Modules
-    - samtools
+    # - samtools
-    - samtools/index
+    # - samtools/index
-    - samtools/sort
+    # - samtools/sort
-    - samtools/stats
+    # - samtools/stats
-    - samtools/idxstats
+    # - samtools/idxstats
-    - samtools/flagstat
+    # - samtools/flagstat
   files:
     - path: ./output/samtools/test.sorted.bam
       md5sum: e4c77897d6824ce4df486d1b100618af
@@ -28,12 +28,12 @@
     - subworkflows/bam_sort_samtools
     - subworkflows/bam_stats_samtools
     # Modules
-    - samtools
+    # - samtools
-    - samtools/index
+    # - samtools/index
-    - samtools/sort
+    # - samtools/sort
-    - samtools/stats
+    # - samtools/stats
-    - samtools/idxstats
+    # - samtools/idxstats
-    - samtools/flagstat
+    # - samtools/flagstat
   files:
     - path: ./output/samtools/test.sorted.bam
       md5sum: bbb2db225f140e69a4ac577f74ccc90f

tests/subworkflows/nf-core/gatk_create_som_pon/test.yml

@@ -2,11 +2,10 @@
   command: nextflow run ./tests/subworkflows/nf-core/gatk_create_som_pon -entry test_gatk_create_som_pon -c tests/config/nextflow.config
   tags:
     - subworkflows/gatk_create_som_pon
+    - gatk4
     # Modules
-    - gatk4
+    # - gatk4/genomicsdbimport
-    - gatk4
+    # - gatk4/createsomaticpanelofnormals
-    - gatk4/genomicsdbimport
-    - gatk4/createsomaticpanelofnormals
   files:
       # gatk4 mutect2
     - path: output/gatk4/test1.vcf.gz