mirror of
https://github.com/MillironX/nf-core_modules.git
synced 2024-11-13 05:13:09 +00:00
Fix subworkflows seperate from modules (#933)
* ci: Remove pytest_subworkflows * ci(bam_sort_samtools): Depend on paths-filter instead of pytest-workflow Co-authored-by: Harshil Patel <drpatelhh@gmail.com> Co-authored-by: Gregor Sturm <mail@gregor-sturm.de> Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com> * ci: Revert back to one job branch * ci(align_bowtie2): Run tests that depend on bam_sort_samtools * ci: Fix anchor not being created yet * ci: Update sra_fastq tags and pytest_modules * fix(bam_sort_samtools): Update nextflow.config with params * test(subworkflows): Update gatk_create_som_pon tags * ci: Point to subworkflow_hacks branch of nf-core tools Co-authored-by: Harshil Patel <drpatelhh@gmail.com> Co-authored-by: Gregor Sturm <mail@gregor-sturm.de> Co-authored-by: Jose Espinosa-Carrasco <kadomu@gmail.com>
This commit is contained in:
Edmund Miller
GitHub
parent
527ccdb419
commit
2af071ed0d
9 changed files with 77 additions and 177 deletions
2
.github/workflows/nf-core-linting.yml
vendored
2
.github/workflows/nf-core-linting.yml
vendored
|
|
@ -60,7 +60,7 @@ jobs:
|
|||
|
||||
# FIXME: Remove this when nf-core modules lint stabilizes and install stable release
|
||||
- name: Install nf-core tools development version
|
||||
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
|
||||
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@subworkflow_hacks
|
||||
|
||||
- name: Install Nextflow
|
||||
env:
|
||||
|
|
|
|||
118
.github/workflows/pytest-workflow.yml
vendored
118
.github/workflows/pytest-workflow.yml
vendored
|
|
@ -6,11 +6,8 @@ on:
|
|||
branches: [master]
|
||||
|
||||
jobs:
|
||||
###########
|
||||
# Modules #
|
||||
###########
|
||||
module_changes:
|
||||
name: Check for changes in the modules
|
||||
changes:
|
||||
name: Check for changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
# Expose matched filters as job 'modules' output variable
|
||||
|
|
@ -23,120 +20,17 @@ jobs:
|
|||
with:
|
||||
filters: "tests/config/pytest_modules.yml"
|
||||
|
||||
module_test:
|
||||
test:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
|
||||
needs: module_changes
|
||||
if: needs.module_changes.outputs.modules != '[]'
|
||||
needs: changes
|
||||
if: needs.changes.outputs.modules != '[]'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
nxf_version: ["21.04.0"]
|
||||
tags: ${{ fromJson(needs.module_changes.outputs.modules) }}
|
||||
profile: ["docker", "singularity", "conda"]
|
||||
env:
|
||||
NXF_ANSI_LOG: false
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: python -m pip install --upgrade pip pytest-workflow
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: /usr/local/bin/nextflow
|
||||
key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-nextflow-
|
||||
|
||||
- name: Install Nextflow
|
||||
env:
|
||||
NXF_VER: ${{ matrix.nxf_version }}
|
||||
CAPSULE_LOG: none
|
||||
run: |
|
||||
wget -qO- get.nextflow.io | bash
|
||||
sudo mv nextflow /usr/local/bin/
|
||||
|
||||
- name: Set up Singularity
|
||||
if: matrix.profile == 'singularity'
|
||||
uses: eWaterCycle/setup-singularity@v5
|
||||
with:
|
||||
singularity-version: 3.7.1
|
||||
|
||||
- name: Setup miniconda
|
||||
if: matrix.profile == 'conda'
|
||||
uses: conda-incubator/setup-miniconda@v2
|
||||
with:
|
||||
auto-update-conda: true
|
||||
channels: conda-forge,bioconda,defaults
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Conda clean
|
||||
if: matrix.profile == 'conda'
|
||||
run: conda clean -a
|
||||
|
||||
# Test the module
|
||||
- name: Run pytest-workflow
|
||||
# only use one thread for pytest-workflow to avoid race condition on conda cache.
|
||||
run: NF_CORE_MODULES_TEST=1 TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: logs-${{ matrix.profile }}-${{ matrix.nxf_version }}
|
||||
path: |
|
||||
/home/runner/pytest_workflow_*/*/.nextflow.log
|
||||
/home/runner/pytest_workflow_*/*/log.out
|
||||
/home/runner/pytest_workflow_*/*/log.err
|
||||
/home/runner/pytest_workflow_*/*/work
|
||||
|
||||
################
|
||||
# Subworkflows #
|
||||
################
|
||||
subworkflow_changes:
|
||||
name: Check for changes in the subworkflows
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
# Expose matched filters as job 'subworkflows' output variable
|
||||
subworkflows: ${{ steps.filter.outputs.changes }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: dorny/paths-filter@v2
|
||||
id: filter
|
||||
with:
|
||||
filters: "tests/config/pytest_subworkflows.yml"
|
||||
|
||||
subworkflow_test:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }}
|
||||
needs: subworkflow_changes
|
||||
if: needs.subworkflow_changes.outputs.subworkflows != '[]'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
nxf_version: ["21.04.0"]
|
||||
tags: ${{ fromJson(needs.subworkflow_changes.outputs.subworkflows) }}
|
||||
tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
|
||||
profile: ["docker", "singularity", "conda"]
|
||||
env:
|
||||
NXF_ANSI_LOG: false
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
params.options = [:]
|
||||
params.sort_options = [:]
|
||||
params.index_options = [:]
|
||||
params.stats_options = [:]
|
||||
|
|
|
|||
|
|
@ -466,7 +466,7 @@ gatk4/createsequencedictionary:
|
|||
- modules/gatk4/createsequencedictionary/**
|
||||
- tests/modules/gatk4/createsequencedictionary/**
|
||||
|
||||
gatk4/createsomaticpanelofnormals:
|
||||
gatk4/createsomaticpanelofnormals: &gatk4/createsomaticpanelofnormals
|
||||
- modules/gatk4/createsomaticpanelofnormals/**
|
||||
- tests/modules/gatk4/createsomaticpanelofnormals/**
|
||||
|
||||
|
|
@ -482,7 +482,7 @@ gatk4/filtermutectcalls:
|
|||
- modules/gatk4/filtermutectcalls/**
|
||||
- tests/modules/gatk4/filtermutectcalls/**
|
||||
|
||||
gatk4/genomicsdbimport:
|
||||
gatk4/genomicsdbimport: &gatk4/genomicsdbimport
|
||||
- modules/gatk4/genomicsdbimport/**
|
||||
- tests/modules/gatk4/genomicsdbimport/**
|
||||
|
||||
|
|
@ -1163,7 +1163,7 @@ samtools/idxstats:
|
|||
- modules/samtools/idxstats/**
|
||||
- tests/modules/samtools/idxstats/**
|
||||
|
||||
samtools/index:
|
||||
samtools/index: &samtools/index
|
||||
- modules/samtools/index/**
|
||||
- tests/modules/samtools/index/**
|
||||
|
||||
|
|
@ -1175,7 +1175,7 @@ samtools/mpileup:
|
|||
- modules/samtools/mpileup/**
|
||||
- tests/modules/samtools/mpileup/**
|
||||
|
||||
samtools/sort:
|
||||
samtools/sort: &samtools/sort
|
||||
- modules/samtools/sort/**
|
||||
- tests/modules/samtools/sort/**
|
||||
|
||||
|
|
@ -1251,11 +1251,11 @@ spatyper:
|
|||
- modules/spatyper/**
|
||||
- tests/modules/spatyper/**
|
||||
|
||||
sratools/fasterqdump:
|
||||
sratools/fasterqdump: &sratools/fasterqdump
|
||||
- modules/sratools/fasterqdump/**
|
||||
- tests/modules/sratools/fasterqdump/**
|
||||
|
||||
sratools/prefetch:
|
||||
sratools/prefetch: &sratools/prefetch
|
||||
- modules/sratools/prefetch/**
|
||||
- tests/modules/sratools/prefetch/**
|
||||
|
||||
|
|
@ -1374,3 +1374,31 @@ yara/index:
|
|||
yara/mapper:
|
||||
- modules/yara/mapper/**
|
||||
- tests/modules/yara/mapper/**
|
||||
|
||||
subworkflows/bam_stats_samtools: &subworkflows/bam_stats_samtools
|
||||
- subworkflows/nf-core/bam_stats_samtools/**
|
||||
- tests/subworkflows/nf-core/bam_stats_samtools/**
|
||||
|
||||
subworkflows/bam_sort_samtools: &subworkflows/bam_sort_samtools
|
||||
- subworkflows/nf-core/bam_sort_samtools/**
|
||||
- tests/subworkflows/nf-core/bam_sort_samtools/**
|
||||
- *samtools/sort
|
||||
- *samtools/index
|
||||
- *subworkflows/bam_stats_samtools
|
||||
|
||||
subworkflows/align_bowtie2:
|
||||
- subworkflows/nf-core/align_bowtie2/**
|
||||
- tests/subworkflows/nf-core/align_bowtie2/**
|
||||
- *subworkflows/bam_sort_samtools
|
||||
|
||||
subworkflows/sra_fastq:
|
||||
- subworkflows/nf-core/sra_fastq/**
|
||||
- tests/subworkflows/nf-core/sra_fastq/**
|
||||
- *sratools/fasterqdump
|
||||
- *sratools/prefetch
|
||||
|
||||
subworkflows/gatk_create_som_pon:
|
||||
- subworkflows/nf-core/gatk_create_som_pon/**
|
||||
- tests/subworkflows/nf-core/gatk_create_som_pon/**
|
||||
- *gatk4/genomicsdbimport
|
||||
- *gatk4/createsomaticpanelofnormals
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
subworkflows/align_bowtie2:
|
||||
- subworkflows/nf-core/align_bowtie2/**
|
||||
- tests/subworkflows/nf-core/align_bowtie2/**
|
||||
|
||||
subworkflows/bam_stats_samtools:
|
||||
- subworkflows/nf-core/bam_stats_samtools/**
|
||||
- tests/subworkflows/nf-core/bam_stats_samtools/**
|
||||
|
||||
subworkflows/bam_sort_samtools:
|
||||
- subworkflows/nf-core/bam_sort_samtools/**
|
||||
- tests/subworkflows/nf-core/bam_sort_samtools/**
|
||||
|
||||
subworkflows/sra_fastq:
|
||||
- subworkflows/nf-core/sra_fastq/**
|
||||
- tests/subworkflows/nf-core/sra_fastq/**
|
||||
|
||||
subworkflows/gatk_create_som_pon:
|
||||
- subworkflows/nf-core/gatk_create_som_pon/**
|
||||
- tests/subworkflows/nf-core/gatk_create_som_pon/**
|
||||
|
||||
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
- name: sratools fasterqdump test_sratools_fasterqdump_single_end
|
||||
command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_single_end -c tests/config/nextflow.config
|
||||
tags:
|
||||
- sratools
|
||||
- sratools/fasterqdump
|
||||
- subworkflows/sra_fastq
|
||||
files:
|
||||
- path: output/sratools/SRR13255544.fastq.gz
|
||||
md5sum: 1054c7b71884acdb5eed8a378f18be82
|
||||
|
|
@ -12,8 +11,7 @@
|
|||
- name: sratools fasterqdump test_sratools_fasterqdump_paired_end
|
||||
command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c tests/config/nextflow.config
|
||||
tags:
|
||||
- sratools
|
||||
- sratools/fasterqdump
|
||||
- subworkflows/sra_fastq
|
||||
files:
|
||||
- path: output/sratools/SRR11140744_1.fastq.gz
|
||||
md5sum: 193809c784a4ea132ab2a253fa4f55b6
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@
|
|||
- subworkflows/bam_sort_samtools
|
||||
- subworkflows/bam_stats_samtools
|
||||
# Modules
|
||||
- bowtie2
|
||||
- bowtie2/align
|
||||
- samtools
|
||||
- samtools/index
|
||||
- samtools/sort
|
||||
- samtools/stats
|
||||
- samtools/idxstats
|
||||
- samtools/flagstat
|
||||
# - bowtie2
|
||||
# - bowtie2/align
|
||||
# - samtools
|
||||
# - samtools/index
|
||||
# - samtools/sort
|
||||
# - samtools/stats
|
||||
# - samtools/idxstats
|
||||
# - samtools/flagstat
|
||||
files:
|
||||
- path: ./output/bowtie2/test.bam
|
||||
- path: ./output/bowtie2/test.bowtie2.log
|
||||
|
|
@ -46,14 +46,14 @@
|
|||
- subworkflows/bam_sort_samtools
|
||||
- subworkflows/bam_stats_samtools
|
||||
# Modules
|
||||
- bowtie2
|
||||
- bowtie2/align
|
||||
- samtools
|
||||
- samtools/index
|
||||
- samtools/sort
|
||||
- samtools/stats
|
||||
- samtools/idxstats
|
||||
- samtools/flagstat
|
||||
# - bowtie2
|
||||
# - bowtie2/align
|
||||
# - samtools
|
||||
# - samtools/index
|
||||
# - samtools/sort
|
||||
# - samtools/stats
|
||||
# - samtools/idxstats
|
||||
# - samtools/flagstat
|
||||
files:
|
||||
- path: ./output/bowtie2/test.bam
|
||||
- path: ./output/bowtie2/test.bowtie2.log
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
- subworkflows/bam_sort_samtools
|
||||
- subworkflows/bam_stats_samtools
|
||||
# Modules
|
||||
- samtools
|
||||
- samtools/index
|
||||
- samtools/sort
|
||||
- samtools/stats
|
||||
- samtools/idxstats
|
||||
- samtools/flagstat
|
||||
# - samtools
|
||||
# - samtools/index
|
||||
# - samtools/sort
|
||||
# - samtools/stats
|
||||
# - samtools/idxstats
|
||||
# - samtools/flagstat
|
||||
files:
|
||||
- path: ./output/samtools/test.sorted.bam
|
||||
md5sum: e4c77897d6824ce4df486d1b100618af
|
||||
|
|
@ -28,12 +28,12 @@
|
|||
- subworkflows/bam_sort_samtools
|
||||
- subworkflows/bam_stats_samtools
|
||||
# Modules
|
||||
- samtools
|
||||
- samtools/index
|
||||
- samtools/sort
|
||||
- samtools/stats
|
||||
- samtools/idxstats
|
||||
- samtools/flagstat
|
||||
# - samtools
|
||||
# - samtools/index
|
||||
# - samtools/sort
|
||||
# - samtools/stats
|
||||
# - samtools/idxstats
|
||||
# - samtools/flagstat
|
||||
files:
|
||||
- path: ./output/samtools/test.sorted.bam
|
||||
md5sum: bbb2db225f140e69a4ac577f74ccc90f
|
||||
|
|
|
|||
|
|
@ -2,11 +2,10 @@
|
|||
command: nextflow run ./tests/subworkflows/nf-core/gatk_create_som_pon -entry test_gatk_create_som_pon -c tests/config/nextflow.config
|
||||
tags:
|
||||
- subworkflows/gatk_create_som_pon
|
||||
- gatk4
|
||||
# Modules
|
||||
- gatk4
|
||||
- gatk4
|
||||
- gatk4/genomicsdbimport
|
||||
- gatk4/createsomaticpanelofnormals
|
||||
# - gatk4/genomicsdbimport
|
||||
# - gatk4/createsomaticpanelofnormals
|
||||
files:
|
||||
# gatk4 mutect2
|
||||
- path: output/gatk4/test1.vcf.gz
|
||||
|
|
|
|||
Loading…
Reference in a new issue