Fix subworkflows seperate from modules (#933)

* ci: Remove pytest_subworkflows

* ci(bam_sort_samtools): Depend on paths-filter instead of pytest-workflow

Co-authored-by: Harshil Patel <drpatelhh@gmail.com>
Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>
Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com>

* ci: Revert back to one job branch

* ci(align_bowtie2): Run tests that depend on bam_sort_samtools

* ci: Fix anchor not being created yet

* ci: Update sra_fastq tags and pytest_modules

* fix(bam_sort_samtools): Update nextflow.config with params

* test(subworkflows): Update gatk_create_som_pon tags

* ci: Point to subworkflow_hacks branch of nf-core tools

Co-authored-by: Harshil Patel <drpatelhh@gmail.com>
Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>
Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com>
This commit is contained in:
Edmund Miller 2021-11-15 12:18:46 -06:00 committed by GitHub
parent 527ccdb419
commit 2af071ed0d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 77 additions and 177 deletions

View file

@ -60,7 +60,7 @@ jobs:
# FIXME: Remove this when nf-core modules lint stabilizes and install stable release
- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@subworkflow_hacks
- name: Install Nextflow
env:

View file

@ -6,11 +6,8 @@ on:
branches: [master]
jobs:
###########
# Modules #
###########
module_changes:
name: Check for changes in the modules
changes:
name: Check for changes
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'modules' output variable
@ -23,120 +20,17 @@ jobs:
with:
filters: "tests/config/pytest_modules.yml"
module_test:
test:
runs-on: ubuntu-20.04
name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
needs: module_changes
if: needs.module_changes.outputs.modules != '[]'
needs: changes
if: needs.changes.outputs.modules != '[]'
strategy:
fail-fast: false
matrix:
nxf_version: ["21.04.0"]
tags: ${{ fromJson(needs.module_changes.outputs.modules) }}
profile: ["docker", "singularity", "conda"]
env:
NXF_ANSI_LOG: false
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.x"
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.x"
- name: Install Python dependencies
run: python -m pip install --upgrade pip pytest-workflow
- uses: actions/cache@v2
with:
path: /usr/local/bin/nextflow
key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }}
restore-keys: |
${{ runner.os }}-nextflow-
- name: Install Nextflow
env:
NXF_VER: ${{ matrix.nxf_version }}
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Set up Singularity
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-singularity@v5
with:
singularity-version: 3.7.1
- name: Setup miniconda
if: matrix.profile == 'conda'
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Conda clean
if: matrix.profile == 'conda'
run: conda clean -a
# Test the module
- name: Run pytest-workflow
# only use one thread for pytest-workflow to avoid race condition on conda cache.
run: NF_CORE_MODULES_TEST=1 TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.profile }}-${{ matrix.nxf_version }}
path: |
/home/runner/pytest_workflow_*/*/.nextflow.log
/home/runner/pytest_workflow_*/*/log.out
/home/runner/pytest_workflow_*/*/log.err
/home/runner/pytest_workflow_*/*/work
################
# Subworkflows #
################
subworkflow_changes:
name: Check for changes in the subworkflows
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'subworkflows' output variable
subworkflows: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v2
- uses: dorny/paths-filter@v2
id: filter
with:
filters: "tests/config/pytest_subworkflows.yml"
subworkflow_test:
runs-on: ubuntu-20.04
name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
needs: subworkflow_changes
if: needs.subworkflow_changes.outputs.subworkflows != '[]'
strategy:
fail-fast: false
matrix:
nxf_version: ["21.04.0"]
tags: ${{ fromJson(needs.subworkflow_changes.outputs.subworkflows) }}
tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
profile: ["docker", "singularity", "conda"]
env:
NXF_ANSI_LOG: false

View file

@ -1 +1,3 @@
params.options = [:]
params.sort_options = [:]
params.index_options = [:]
params.stats_options = [:]

View file

@ -466,7 +466,7 @@ gatk4/createsequencedictionary:
- modules/gatk4/createsequencedictionary/**
- tests/modules/gatk4/createsequencedictionary/**
gatk4/createsomaticpanelofnormals:
gatk4/createsomaticpanelofnormals: &gatk4/createsomaticpanelofnormals
- modules/gatk4/createsomaticpanelofnormals/**
- tests/modules/gatk4/createsomaticpanelofnormals/**
@ -482,7 +482,7 @@ gatk4/filtermutectcalls:
- modules/gatk4/filtermutectcalls/**
- tests/modules/gatk4/filtermutectcalls/**
gatk4/genomicsdbimport:
gatk4/genomicsdbimport: &gatk4/genomicsdbimport
- modules/gatk4/genomicsdbimport/**
- tests/modules/gatk4/genomicsdbimport/**
@ -1163,7 +1163,7 @@ samtools/idxstats:
- modules/samtools/idxstats/**
- tests/modules/samtools/idxstats/**
samtools/index:
samtools/index: &samtools/index
- modules/samtools/index/**
- tests/modules/samtools/index/**
@ -1175,7 +1175,7 @@ samtools/mpileup:
- modules/samtools/mpileup/**
- tests/modules/samtools/mpileup/**
samtools/sort:
samtools/sort: &samtools/sort
- modules/samtools/sort/**
- tests/modules/samtools/sort/**
@ -1251,11 +1251,11 @@ spatyper:
- modules/spatyper/**
- tests/modules/spatyper/**
sratools/fasterqdump:
sratools/fasterqdump: &sratools/fasterqdump
- modules/sratools/fasterqdump/**
- tests/modules/sratools/fasterqdump/**
sratools/prefetch:
sratools/prefetch: &sratools/prefetch
- modules/sratools/prefetch/**
- tests/modules/sratools/prefetch/**
@ -1374,3 +1374,31 @@ yara/index:
yara/mapper:
- modules/yara/mapper/**
- tests/modules/yara/mapper/**
subworkflows/bam_stats_samtools: &subworkflows/bam_stats_samtools
- subworkflows/nf-core/bam_stats_samtools/**
- tests/subworkflows/nf-core/bam_stats_samtools/**
subworkflows/bam_sort_samtools: &subworkflows/bam_sort_samtools
- subworkflows/nf-core/bam_sort_samtools/**
- tests/subworkflows/nf-core/bam_sort_samtools/**
- *samtools/sort
- *samtools/index
- *subworkflows/bam_stats_samtools
subworkflows/align_bowtie2:
- subworkflows/nf-core/align_bowtie2/**
- tests/subworkflows/nf-core/align_bowtie2/**
- *subworkflows/bam_sort_samtools
subworkflows/sra_fastq:
- subworkflows/nf-core/sra_fastq/**
- tests/subworkflows/nf-core/sra_fastq/**
- *sratools/fasterqdump
- *sratools/prefetch
subworkflows/gatk_create_som_pon:
- subworkflows/nf-core/gatk_create_som_pon/**
- tests/subworkflows/nf-core/gatk_create_som_pon/**
- *gatk4/genomicsdbimport
- *gatk4/createsomaticpanelofnormals

View file

@ -1,21 +0,0 @@
subworkflows/align_bowtie2:
- subworkflows/nf-core/align_bowtie2/**
- tests/subworkflows/nf-core/align_bowtie2/**
subworkflows/bam_stats_samtools:
- subworkflows/nf-core/bam_stats_samtools/**
- tests/subworkflows/nf-core/bam_stats_samtools/**
subworkflows/bam_sort_samtools:
- subworkflows/nf-core/bam_sort_samtools/**
- tests/subworkflows/nf-core/bam_sort_samtools/**
subworkflows/sra_fastq:
- subworkflows/nf-core/sra_fastq/**
- tests/subworkflows/nf-core/sra_fastq/**
subworkflows/gatk_create_som_pon:
- subworkflows/nf-core/gatk_create_som_pon/**
- tests/subworkflows/nf-core/gatk_create_som_pon/**

View file

@ -1,8 +1,7 @@
- name: sratools fasterqdump test_sratools_fasterqdump_single_end
command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_single_end -c tests/config/nextflow.config
tags:
- sratools
- sratools/fasterqdump
- subworkflows/sra_fastq
files:
- path: output/sratools/SRR13255544.fastq.gz
md5sum: 1054c7b71884acdb5eed8a378f18be82
@ -12,8 +11,7 @@
- name: sratools fasterqdump test_sratools_fasterqdump_paired_end
command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c tests/config/nextflow.config
tags:
- sratools
- sratools/fasterqdump
- subworkflows/sra_fastq
files:
- path: output/sratools/SRR11140744_1.fastq.gz
md5sum: 193809c784a4ea132ab2a253fa4f55b6

View file

@ -5,14 +5,14 @@
- subworkflows/bam_sort_samtools
- subworkflows/bam_stats_samtools
# Modules
- bowtie2
- bowtie2/align
- samtools
- samtools/index
- samtools/sort
- samtools/stats
- samtools/idxstats
- samtools/flagstat
# - bowtie2
# - bowtie2/align
# - samtools
# - samtools/index
# - samtools/sort
# - samtools/stats
# - samtools/idxstats
# - samtools/flagstat
files:
- path: ./output/bowtie2/test.bam
- path: ./output/bowtie2/test.bowtie2.log
@ -46,14 +46,14 @@
- subworkflows/bam_sort_samtools
- subworkflows/bam_stats_samtools
# Modules
- bowtie2
- bowtie2/align
- samtools
- samtools/index
- samtools/sort
- samtools/stats
- samtools/idxstats
- samtools/flagstat
# - bowtie2
# - bowtie2/align
# - samtools
# - samtools/index
# - samtools/sort
# - samtools/stats
# - samtools/idxstats
# - samtools/flagstat
files:
- path: ./output/bowtie2/test.bam
- path: ./output/bowtie2/test.bowtie2.log

View file

@ -4,12 +4,12 @@
- subworkflows/bam_sort_samtools
- subworkflows/bam_stats_samtools
# Modules
- samtools
- samtools/index
- samtools/sort
- samtools/stats
- samtools/idxstats
- samtools/flagstat
# - samtools
# - samtools/index
# - samtools/sort
# - samtools/stats
# - samtools/idxstats
# - samtools/flagstat
files:
- path: ./output/samtools/test.sorted.bam
md5sum: e4c77897d6824ce4df486d1b100618af
@ -28,12 +28,12 @@
- subworkflows/bam_sort_samtools
- subworkflows/bam_stats_samtools
# Modules
- samtools
- samtools/index
- samtools/sort
- samtools/stats
- samtools/idxstats
- samtools/flagstat
# - samtools
# - samtools/index
# - samtools/sort
# - samtools/stats
# - samtools/idxstats
# - samtools/flagstat
files:
- path: ./output/samtools/test.sorted.bam
md5sum: bbb2db225f140e69a4ac577f74ccc90f

View file

@ -2,11 +2,10 @@
command: nextflow run ./tests/subworkflows/nf-core/gatk_create_som_pon -entry test_gatk_create_som_pon -c tests/config/nextflow.config
tags:
- subworkflows/gatk_create_som_pon
- gatk4
# Modules
- gatk4
- gatk4
- gatk4/genomicsdbimport
- gatk4/createsomaticpanelofnormals
# - gatk4/genomicsdbimport
# - gatk4/createsomaticpanelofnormals
files:
# gatk4 mutect2
- path: output/gatk4/test1.vcf.gz