From 2af071ed0d70ec7fa25b08e3fa48ea6fd9564a38 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 15 Nov 2021 12:18:46 -0600 Subject: [PATCH] Fix subworkflows seperate from modules (#933) * ci: Remove pytest_subworkflows * ci(bam_sort_samtools): Depend on paths-filter instead of pytest-workflow Co-authored-by: Harshil Patel Co-authored-by: Gregor Sturm Co-authored-by: Jose Espinosa-Carrasco * ci: Revert back to one job branch * ci(align_bowtie2): Run tests that depend on bam_sort_samtools * ci: Fix anchor not being created yet * ci: Update sra_fastq tags and pytest_modules * fix(bam_sort_samtools): Update nextflow.config with params * test(subworkflows): Update gatk_create_som_pon tags * ci: Point to subworkflow_hacks branch of nf-core tools Co-authored-by: Harshil Patel Co-authored-by: Gregor Sturm Co-authored-by: Jose Espinosa-Carrasco --- .github/workflows/nf-core-linting.yml | 2 +- .github/workflows/pytest-workflow.yml | 118 +----------------- .../nf-core/bam_sort_samtools/nextflow.config | 4 +- tests/config/pytest_modules.yml | 40 +++++- tests/config/pytest_subworkflows.yml | 21 ---- tests/modules/sratools/fasterqdump/test.yml | 6 +- .../nf-core/align_bowtie2/test.yml | 32 ++--- .../nf-core/bam_sort_samtools/test.yml | 24 ++-- .../nf-core/gatk_create_som_pon/test.yml | 7 +- 9 files changed, 77 insertions(+), 177 deletions(-) delete mode 100644 tests/config/pytest_subworkflows.yml diff --git a/.github/workflows/nf-core-linting.yml b/.github/workflows/nf-core-linting.yml index 55b8c296..ce441413 100644 --- a/.github/workflows/nf-core-linting.yml +++ b/.github/workflows/nf-core-linting.yml @@ -60,7 +60,7 @@ jobs: # FIXME: Remove this when nf-core modules lint stabilizes and install stable release - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@subworkflow_hacks - name: Install Nextflow env: diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 0b509527..7cbb2689 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -6,11 +6,8 @@ on: branches: [master] jobs: - ########### - # Modules # - ########### - module_changes: - name: Check for changes in the modules + changes: + name: Check for changes runs-on: ubuntu-latest outputs: # Expose matched filters as job 'modules' output variable @@ -23,120 +20,17 @@ jobs: with: filters: "tests/config/pytest_modules.yml" - module_test: + test: runs-on: ubuntu-20.04 name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }} - needs: module_changes - if: needs.module_changes.outputs.modules != '[]' + needs: changes + if: needs.changes.outputs.modules != '[]' strategy: fail-fast: false matrix: nxf_version: ["21.04.0"] - tags: ${{ fromJson(needs.module_changes.outputs.modules) }} - profile: ["docker", "singularity", "conda"] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow - - - uses: actions/cache@v2 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Setup miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: NF_CORE_MODULES_TEST=1 TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.profile }}-${{ matrix.nxf_version }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - - ################ - # Subworkflows # - ################ - subworkflow_changes: - name: Check for changes in the subworkflows - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'subworkflows' output variable - subworkflows: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v2 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_subworkflows.yml" - - subworkflow_test: - runs-on: ubuntu-20.04 - - name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }} - needs: subworkflow_changes - if: needs.subworkflow_changes.outputs.subworkflows != '[]' - strategy: - fail-fast: false - matrix: - nxf_version: ["21.04.0"] - tags: ${{ fromJson(needs.subworkflow_changes.outputs.subworkflows) }} + tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] profile: ["docker", "singularity", "conda"] env: NXF_ANSI_LOG: false diff --git a/subworkflows/nf-core/bam_sort_samtools/nextflow.config b/subworkflows/nf-core/bam_sort_samtools/nextflow.config index 2fd55747..72128aad 100644 --- a/subworkflows/nf-core/bam_sort_samtools/nextflow.config +++ b/subworkflows/nf-core/bam_sort_samtools/nextflow.config @@ -1 +1,3 @@ -params.options = [:] +params.sort_options = [:] +params.index_options = [:] +params.stats_options = [:] diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index d925b76d..baaee3b8 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -466,7 +466,7 @@ gatk4/createsequencedictionary: - modules/gatk4/createsequencedictionary/** - tests/modules/gatk4/createsequencedictionary/** -gatk4/createsomaticpanelofnormals: +gatk4/createsomaticpanelofnormals: &gatk4/createsomaticpanelofnormals - modules/gatk4/createsomaticpanelofnormals/** - tests/modules/gatk4/createsomaticpanelofnormals/** @@ -482,7 +482,7 @@ gatk4/filtermutectcalls: - modules/gatk4/filtermutectcalls/** - tests/modules/gatk4/filtermutectcalls/** -gatk4/genomicsdbimport: +gatk4/genomicsdbimport: &gatk4/genomicsdbimport - modules/gatk4/genomicsdbimport/** - tests/modules/gatk4/genomicsdbimport/** @@ -1163,7 +1163,7 @@ samtools/idxstats: - modules/samtools/idxstats/** - tests/modules/samtools/idxstats/** -samtools/index: +samtools/index: &samtools/index - modules/samtools/index/** - tests/modules/samtools/index/** @@ -1175,7 +1175,7 @@ samtools/mpileup: - modules/samtools/mpileup/** - tests/modules/samtools/mpileup/** -samtools/sort: +samtools/sort: &samtools/sort - modules/samtools/sort/** - tests/modules/samtools/sort/** @@ -1251,11 +1251,11 @@ spatyper: - modules/spatyper/** - tests/modules/spatyper/** -sratools/fasterqdump: +sratools/fasterqdump: &sratools/fasterqdump - modules/sratools/fasterqdump/** - tests/modules/sratools/fasterqdump/** -sratools/prefetch: +sratools/prefetch: &sratools/prefetch - modules/sratools/prefetch/** - tests/modules/sratools/prefetch/** @@ -1374,3 +1374,31 @@ yara/index: yara/mapper: - modules/yara/mapper/** - tests/modules/yara/mapper/** + +subworkflows/bam_stats_samtools: &subworkflows/bam_stats_samtools + - subworkflows/nf-core/bam_stats_samtools/** + - tests/subworkflows/nf-core/bam_stats_samtools/** + +subworkflows/bam_sort_samtools: &subworkflows/bam_sort_samtools + - subworkflows/nf-core/bam_sort_samtools/** + - tests/subworkflows/nf-core/bam_sort_samtools/** + - *samtools/sort + - *samtools/index + - *subworkflows/bam_stats_samtools + +subworkflows/align_bowtie2: + - subworkflows/nf-core/align_bowtie2/** + - tests/subworkflows/nf-core/align_bowtie2/** + - *subworkflows/bam_sort_samtools + +subworkflows/sra_fastq: + - subworkflows/nf-core/sra_fastq/** + - tests/subworkflows/nf-core/sra_fastq/** + - *sratools/fasterqdump + - *sratools/prefetch + +subworkflows/gatk_create_som_pon: + - subworkflows/nf-core/gatk_create_som_pon/** + - tests/subworkflows/nf-core/gatk_create_som_pon/** + - *gatk4/genomicsdbimport + - *gatk4/createsomaticpanelofnormals diff --git a/tests/config/pytest_subworkflows.yml b/tests/config/pytest_subworkflows.yml deleted file mode 100644 index 4f9c5514..00000000 --- a/tests/config/pytest_subworkflows.yml +++ /dev/null @@ -1,21 +0,0 @@ -subworkflows/align_bowtie2: - - subworkflows/nf-core/align_bowtie2/** - - tests/subworkflows/nf-core/align_bowtie2/** - -subworkflows/bam_stats_samtools: - - subworkflows/nf-core/bam_stats_samtools/** - - tests/subworkflows/nf-core/bam_stats_samtools/** - -subworkflows/bam_sort_samtools: - - subworkflows/nf-core/bam_sort_samtools/** - - tests/subworkflows/nf-core/bam_sort_samtools/** - -subworkflows/sra_fastq: - - subworkflows/nf-core/sra_fastq/** - - tests/subworkflows/nf-core/sra_fastq/** - -subworkflows/gatk_create_som_pon: - - subworkflows/nf-core/gatk_create_som_pon/** - - tests/subworkflows/nf-core/gatk_create_som_pon/** - - \ No newline at end of file diff --git a/tests/modules/sratools/fasterqdump/test.yml b/tests/modules/sratools/fasterqdump/test.yml index 94da4ed8..7d022a0d 100644 --- a/tests/modules/sratools/fasterqdump/test.yml +++ b/tests/modules/sratools/fasterqdump/test.yml @@ -1,8 +1,7 @@ - name: sratools fasterqdump test_sratools_fasterqdump_single_end command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_single_end -c tests/config/nextflow.config tags: - - sratools - - sratools/fasterqdump + - subworkflows/sra_fastq files: - path: output/sratools/SRR13255544.fastq.gz md5sum: 1054c7b71884acdb5eed8a378f18be82 @@ -12,8 +11,7 @@ - name: sratools fasterqdump test_sratools_fasterqdump_paired_end command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c tests/config/nextflow.config tags: - - sratools - - sratools/fasterqdump + - subworkflows/sra_fastq files: - path: output/sratools/SRR11140744_1.fastq.gz md5sum: 193809c784a4ea132ab2a253fa4f55b6 diff --git a/tests/subworkflows/nf-core/align_bowtie2/test.yml b/tests/subworkflows/nf-core/align_bowtie2/test.yml index 51261a14..116ea961 100644 --- a/tests/subworkflows/nf-core/align_bowtie2/test.yml +++ b/tests/subworkflows/nf-core/align_bowtie2/test.yml @@ -5,14 +5,14 @@ - subworkflows/bam_sort_samtools - subworkflows/bam_stats_samtools # Modules - - bowtie2 - - bowtie2/align - - samtools - - samtools/index - - samtools/sort - - samtools/stats - - samtools/idxstats - - samtools/flagstat + # - bowtie2 + # - bowtie2/align + # - samtools + # - samtools/index + # - samtools/sort + # - samtools/stats + # - samtools/idxstats + # - samtools/flagstat files: - path: ./output/bowtie2/test.bam - path: ./output/bowtie2/test.bowtie2.log @@ -46,14 +46,14 @@ - subworkflows/bam_sort_samtools - subworkflows/bam_stats_samtools # Modules - - bowtie2 - - bowtie2/align - - samtools - - samtools/index - - samtools/sort - - samtools/stats - - samtools/idxstats - - samtools/flagstat + # - bowtie2 + # - bowtie2/align + # - samtools + # - samtools/index + # - samtools/sort + # - samtools/stats + # - samtools/idxstats + # - samtools/flagstat files: - path: ./output/bowtie2/test.bam - path: ./output/bowtie2/test.bowtie2.log diff --git a/tests/subworkflows/nf-core/bam_sort_samtools/test.yml b/tests/subworkflows/nf-core/bam_sort_samtools/test.yml index e2fc27d8..88ea9d5a 100644 --- a/tests/subworkflows/nf-core/bam_sort_samtools/test.yml +++ b/tests/subworkflows/nf-core/bam_sort_samtools/test.yml @@ -4,12 +4,12 @@ - subworkflows/bam_sort_samtools - subworkflows/bam_stats_samtools # Modules - - samtools - - samtools/index - - samtools/sort - - samtools/stats - - samtools/idxstats - - samtools/flagstat + # - samtools + # - samtools/index + # - samtools/sort + # - samtools/stats + # - samtools/idxstats + # - samtools/flagstat files: - path: ./output/samtools/test.sorted.bam md5sum: e4c77897d6824ce4df486d1b100618af @@ -28,12 +28,12 @@ - subworkflows/bam_sort_samtools - subworkflows/bam_stats_samtools # Modules - - samtools - - samtools/index - - samtools/sort - - samtools/stats - - samtools/idxstats - - samtools/flagstat + # - samtools + # - samtools/index + # - samtools/sort + # - samtools/stats + # - samtools/idxstats + # - samtools/flagstat files: - path: ./output/samtools/test.sorted.bam md5sum: bbb2db225f140e69a4ac577f74ccc90f diff --git a/tests/subworkflows/nf-core/gatk_create_som_pon/test.yml b/tests/subworkflows/nf-core/gatk_create_som_pon/test.yml index a4478044..e6d80409 100644 --- a/tests/subworkflows/nf-core/gatk_create_som_pon/test.yml +++ b/tests/subworkflows/nf-core/gatk_create_som_pon/test.yml @@ -2,11 +2,10 @@ command: nextflow run ./tests/subworkflows/nf-core/gatk_create_som_pon -entry test_gatk_create_som_pon -c tests/config/nextflow.config tags: - subworkflows/gatk_create_som_pon + - gatk4 # Modules - - gatk4 - - gatk4 - - gatk4/genomicsdbimport - - gatk4/createsomaticpanelofnormals + # - gatk4/genomicsdbimport + # - gatk4/createsomaticpanelofnormals files: # gatk4 mutect2 - path: output/gatk4/test1.vcf.gz