From a1f4d53d3e7898f62928ef401359273120831ec2 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 11:16:19 -0600 Subject: [PATCH 01/22] ci: Upload logs on failures --- .github/workflows/bowtie_align.yml | 53 +++++++++++-------- .github/workflows/bowtie_index.yml | 9 ++++ .github/workflows/bwa_index.yml | 9 ++++ .github/workflows/bwa_mem.yml | 9 ++++ .github/workflows/fastqc.yml | 9 ++++ .github/workflows/gffread.yml | 9 ++++ .github/workflows/multiqc.yml | 9 ++++ .../picard_collectmultiplemetrics.yml | 9 ++++ .github/workflows/picard_markduplicates.yml | 9 ++++ .github/workflows/picard_mergesamfiles.yml | 9 ++++ .github/workflows/preseq_lcextrap.yml | 9 ++++ .github/workflows/samtools_flagstat.yml | 9 ++++ .github/workflows/samtools_idxstats.yml | 9 ++++ .github/workflows/samtools_index.yml | 9 ++++ .github/workflows/samtools_sort.yml | 9 ++++ .github/workflows/samtools_stats.yml | 9 ++++ .github/workflows/stringtie.yml | 9 ++++ .github/workflows/trimgalore.yml | 9 ++++ .github/workflows/ucsc_bedgraphtobigwig.yml | 9 ++++ 19 files changed, 193 insertions(+), 22 deletions(-) diff --git a/.github/workflows/bowtie_align.yml b/.github/workflows/bowtie_align.yml index 8b9a19fe..14222a7b 100644 --- a/.github/workflows/bowtie_align.yml +++ b/.github/workflows/bowtie_align.yml @@ -14,29 +14,38 @@ on: - tests/software/bowtie/** jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow + # Test the module + - run: pytest --tag bowtie_align --symlink --wt 2 - # Test the module - - run: pytest --tag bowtie_align --symlink --wt 2 + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bowtie_index.yml b/.github/workflows/bowtie_index.yml index 291d04f5..4fc29026 100644 --- a/.github/workflows/bowtie_index.yml +++ b/.github/workflows/bowtie_index.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag bowtie_index --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_index.yml b/.github/workflows/bwa_index.yml index 06659057..5156965d 100644 --- a/.github/workflows/bwa_index.yml +++ b/.github/workflows/bwa_index.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag bwa_index --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_mem.yml b/.github/workflows/bwa_mem.yml index 69b737af..ba3f6609 100644 --- a/.github/workflows/bwa_mem.yml +++ b/.github/workflows/bwa_mem.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag bwa_index --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/fastqc.yml b/.github/workflows/fastqc.yml index 3de0b9a7..59d324ee 100644 --- a/.github/workflows/fastqc.yml +++ b/.github/workflows/fastqc.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag fastqc --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/gffread.yml b/.github/workflows/gffread.yml index 37312af5..3ff64058 100644 --- a/.github/workflows/gffread.yml +++ b/.github/workflows/gffread.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag gffread --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/multiqc.yml b/.github/workflows/multiqc.yml index ea6466ab..8de15621 100644 --- a/.github/workflows/multiqc.yml +++ b/.github/workflows/multiqc.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag multiqc --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_collectmultiplemetrics.yml b/.github/workflows/picard_collectmultiplemetrics.yml index 37c0d5a1..ecccdf70 100644 --- a/.github/workflows/picard_collectmultiplemetrics.yml +++ b/.github/workflows/picard_collectmultiplemetrics.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag picard_collectmultiplemetrics --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_markduplicates.yml b/.github/workflows/picard_markduplicates.yml index 9334ec1f..02b93a0c 100644 --- a/.github/workflows/picard_markduplicates.yml +++ b/.github/workflows/picard_markduplicates.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag picard_markduplicates --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_mergesamfiles.yml b/.github/workflows/picard_mergesamfiles.yml index 1ea5e20c..6aedb733 100644 --- a/.github/workflows/picard_mergesamfiles.yml +++ b/.github/workflows/picard_mergesamfiles.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag picard_mergesamfiles --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/preseq_lcextrap.yml b/.github/workflows/preseq_lcextrap.yml index 33443f5e..997741e1 100644 --- a/.github/workflows/preseq_lcextrap.yml +++ b/.github/workflows/preseq_lcextrap.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag preseq_lcextrap --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_flagstat.yml b/.github/workflows/samtools_flagstat.yml index 441f66a1..08106404 100644 --- a/.github/workflows/samtools_flagstat.yml +++ b/.github/workflows/samtools_flagstat.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag samtools_flagstat --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_idxstats.yml b/.github/workflows/samtools_idxstats.yml index d2d5c49a..ab13eecf 100644 --- a/.github/workflows/samtools_idxstats.yml +++ b/.github/workflows/samtools_idxstats.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag samtools_idxstats --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_index.yml b/.github/workflows/samtools_index.yml index 1a3cdea5..0dcbd744 100644 --- a/.github/workflows/samtools_index.yml +++ b/.github/workflows/samtools_index.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag samtools_index --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_sort.yml b/.github/workflows/samtools_sort.yml index 58f1fec1..5725ab1a 100644 --- a/.github/workflows/samtools_sort.yml +++ b/.github/workflows/samtools_sort.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag samtools_sort --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_stats.yml b/.github/workflows/samtools_stats.yml index f28f4997..f71ef4c0 100644 --- a/.github/workflows/samtools_stats.yml +++ b/.github/workflows/samtools_stats.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag samtools_stats --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/stringtie.yml b/.github/workflows/stringtie.yml index f650dfae..80e9b498 100644 --- a/.github/workflows/stringtie.yml +++ b/.github/workflows/stringtie.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag stringtie --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/trimgalore.yml b/.github/workflows/trimgalore.yml index 8b201e38..c3192afa 100644 --- a/.github/workflows/trimgalore.yml +++ b/.github/workflows/trimgalore.yml @@ -38,3 +38,12 @@ jobs: # Test the module - run: pytest --tag trimgalore --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/ucsc_bedgraphtobigwig.yml b/.github/workflows/ucsc_bedgraphtobigwig.yml index 11772d78..8e7a9473 100644 --- a/.github/workflows/ucsc_bedgraphtobigwig.yml +++ b/.github/workflows/ucsc_bedgraphtobigwig.yml @@ -37,3 +37,12 @@ jobs: # Test the module - run: pytest --tag ucsc_bedgraphtobigwig --symlink --wt 2 + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err From 73bd2d499e79e1e07e080eab83ff9b1993f0b197 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 11:26:58 -0600 Subject: [PATCH 02/22] ci: Add keep-workflow-wd-on-fail --- .github/workflows/bowtie_align.yml | 2 +- .github/workflows/bowtie_index.yml | 2 +- .github/workflows/bwa_index.yml | 2 +- .github/workflows/bwa_mem.yml | 2 +- .github/workflows/fastqc.yml | 2 +- .github/workflows/gffread.yml | 2 +- .github/workflows/multiqc.yml | 2 +- .github/workflows/picard_collectmultiplemetrics.yml | 2 +- .github/workflows/picard_markduplicates.yml | 2 +- .github/workflows/picard_mergesamfiles.yml | 2 +- .github/workflows/preseq_lcextrap.yml | 2 +- .github/workflows/samtools_flagstat.yml | 2 +- .github/workflows/samtools_idxstats.yml | 2 +- .github/workflows/samtools_index.yml | 2 +- .github/workflows/samtools_sort.yml | 2 +- .github/workflows/samtools_stats.yml | 2 +- .github/workflows/stringtie.yml | 2 +- .github/workflows/trimgalore.yml | 2 +- .github/workflows/ucsc_bedgraphtobigwig.yml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/bowtie_align.yml b/.github/workflows/bowtie_align.yml index 14222a7b..55c025d7 100644 --- a/.github/workflows/bowtie_align.yml +++ b/.github/workflows/bowtie_align.yml @@ -39,7 +39,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag bowtie_align --symlink --wt 2 + - run: pytest --kwdof --tag bowtie_align --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/bowtie_index.yml b/.github/workflows/bowtie_index.yml index 4fc29026..93ff35df 100644 --- a/.github/workflows/bowtie_index.yml +++ b/.github/workflows/bowtie_index.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag bowtie_index --symlink --wt 2 + - run: pytest --kwdof --tag bowtie_index --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/bwa_index.yml b/.github/workflows/bwa_index.yml index 5156965d..95395751 100644 --- a/.github/workflows/bwa_index.yml +++ b/.github/workflows/bwa_index.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag bwa_index --symlink --wt 2 + - run: pytest --kwdof --tag bwa_index --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/bwa_mem.yml b/.github/workflows/bwa_mem.yml index ba3f6609..af5290e5 100644 --- a/.github/workflows/bwa_mem.yml +++ b/.github/workflows/bwa_mem.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag bwa_index --symlink --wt 2 + - run: pytest --kwdof --tag bwa_index --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/fastqc.yml b/.github/workflows/fastqc.yml index 59d324ee..504e12ff 100644 --- a/.github/workflows/fastqc.yml +++ b/.github/workflows/fastqc.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag fastqc --symlink --wt 2 + - run: pytest --kwdof --tag fastqc --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/gffread.yml b/.github/workflows/gffread.yml index 3ff64058..7509eafe 100644 --- a/.github/workflows/gffread.yml +++ b/.github/workflows/gffread.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag gffread --symlink --wt 2 + - run: pytest --kwdof --tag gffread --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/multiqc.yml b/.github/workflows/multiqc.yml index 8de15621..4070a1df 100644 --- a/.github/workflows/multiqc.yml +++ b/.github/workflows/multiqc.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag multiqc --symlink --wt 2 + - run: pytest --kwdof --tag multiqc --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/picard_collectmultiplemetrics.yml b/.github/workflows/picard_collectmultiplemetrics.yml index ecccdf70..86130f75 100644 --- a/.github/workflows/picard_collectmultiplemetrics.yml +++ b/.github/workflows/picard_collectmultiplemetrics.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag picard_collectmultiplemetrics --symlink --wt 2 + - run: pytest --kwdof --tag picard_collectmultiplemetrics --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/picard_markduplicates.yml b/.github/workflows/picard_markduplicates.yml index 02b93a0c..5438d30b 100644 --- a/.github/workflows/picard_markduplicates.yml +++ b/.github/workflows/picard_markduplicates.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag picard_markduplicates --symlink --wt 2 + - run: pytest --kwdof --tag picard_markduplicates --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/picard_mergesamfiles.yml b/.github/workflows/picard_mergesamfiles.yml index 6aedb733..aa8fa1a4 100644 --- a/.github/workflows/picard_mergesamfiles.yml +++ b/.github/workflows/picard_mergesamfiles.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag picard_mergesamfiles --symlink --wt 2 + - run: pytest --kwdof --tag picard_mergesamfiles --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/preseq_lcextrap.yml b/.github/workflows/preseq_lcextrap.yml index 997741e1..ea04bc8b 100644 --- a/.github/workflows/preseq_lcextrap.yml +++ b/.github/workflows/preseq_lcextrap.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag preseq_lcextrap --symlink --wt 2 + - run: pytest --kwdof --tag preseq_lcextrap --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/samtools_flagstat.yml b/.github/workflows/samtools_flagstat.yml index 08106404..3b2a956b 100644 --- a/.github/workflows/samtools_flagstat.yml +++ b/.github/workflows/samtools_flagstat.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag samtools_flagstat --symlink --wt 2 + - run: pytest --kwdof --tag samtools_flagstat --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/samtools_idxstats.yml b/.github/workflows/samtools_idxstats.yml index ab13eecf..ffd4ce41 100644 --- a/.github/workflows/samtools_idxstats.yml +++ b/.github/workflows/samtools_idxstats.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag samtools_idxstats --symlink --wt 2 + - run: pytest --kwdof --tag samtools_idxstats --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/samtools_index.yml b/.github/workflows/samtools_index.yml index 0dcbd744..74ea3e45 100644 --- a/.github/workflows/samtools_index.yml +++ b/.github/workflows/samtools_index.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag samtools_index --symlink --wt 2 + - run: pytest --kwdof --tag samtools_index --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/samtools_sort.yml b/.github/workflows/samtools_sort.yml index 5725ab1a..44b4bc12 100644 --- a/.github/workflows/samtools_sort.yml +++ b/.github/workflows/samtools_sort.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag samtools_sort --symlink --wt 2 + - run: pytest --kwdof --tag samtools_sort --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/samtools_stats.yml b/.github/workflows/samtools_stats.yml index f71ef4c0..ba346e60 100644 --- a/.github/workflows/samtools_stats.yml +++ b/.github/workflows/samtools_stats.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag samtools_stats --symlink --wt 2 + - run: pytest --kwdof --tag samtools_stats --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/stringtie.yml b/.github/workflows/stringtie.yml index 80e9b498..d7bb6fff 100644 --- a/.github/workflows/stringtie.yml +++ b/.github/workflows/stringtie.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag stringtie --symlink --wt 2 + - run: pytest --kwdof --tag stringtie --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/trimgalore.yml b/.github/workflows/trimgalore.yml index c3192afa..5bc54de5 100644 --- a/.github/workflows/trimgalore.yml +++ b/.github/workflows/trimgalore.yml @@ -37,7 +37,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag trimgalore --symlink --wt 2 + - run: pytest --kwdof --tag trimgalore --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} diff --git a/.github/workflows/ucsc_bedgraphtobigwig.yml b/.github/workflows/ucsc_bedgraphtobigwig.yml index 8e7a9473..935ad9aa 100644 --- a/.github/workflows/ucsc_bedgraphtobigwig.yml +++ b/.github/workflows/ucsc_bedgraphtobigwig.yml @@ -36,7 +36,7 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow # Test the module - - run: pytest --tag ucsc_bedgraphtobigwig --symlink --wt 2 + - run: pytest --kwdof --tag ucsc_bedgraphtobigwig --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} From f05134e855e870086e1591d7423551baf57379eb Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 11:49:24 -0600 Subject: [PATCH 03/22] ci: Add nextflow.log --- .github/workflows/bowtie_align.yml | 1 + .github/workflows/bowtie_index.yml | 1 + .github/workflows/bwa_index.yml | 1 + .github/workflows/bwa_mem.yml | 1 + .github/workflows/fastqc.yml | 1 + .github/workflows/gffread.yml | 1 + .github/workflows/multiqc.yml | 1 + .github/workflows/picard_collectmultiplemetrics.yml | 1 + .github/workflows/picard_markduplicates.yml | 1 + .github/workflows/picard_mergesamfiles.yml | 1 + .github/workflows/preseq_lcextrap.yml | 1 + .github/workflows/samtools_flagstat.yml | 1 + .github/workflows/samtools_idxstats.yml | 1 + .github/workflows/samtools_index.yml | 1 + .github/workflows/samtools_sort.yml | 1 + .github/workflows/samtools_stats.yml | 1 + .github/workflows/stringtie.yml | 1 + .github/workflows/trimgalore.yml | 1 + .github/workflows/ucsc_bedgraphtobigwig.yml | 1 + 19 files changed, 19 insertions(+) diff --git a/.github/workflows/bowtie_align.yml b/.github/workflows/bowtie_align.yml index 55c025d7..a5ac590c 100644 --- a/.github/workflows/bowtie_align.yml +++ b/.github/workflows/bowtie_align.yml @@ -47,5 +47,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bowtie_index.yml b/.github/workflows/bowtie_index.yml index 93ff35df..cd1d7501 100644 --- a/.github/workflows/bowtie_index.yml +++ b/.github/workflows/bowtie_index.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_index.yml b/.github/workflows/bwa_index.yml index 95395751..632dc5be 100644 --- a/.github/workflows/bwa_index.yml +++ b/.github/workflows/bwa_index.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_mem.yml b/.github/workflows/bwa_mem.yml index af5290e5..2c5ccbcc 100644 --- a/.github/workflows/bwa_mem.yml +++ b/.github/workflows/bwa_mem.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/fastqc.yml b/.github/workflows/fastqc.yml index 504e12ff..e9fae495 100644 --- a/.github/workflows/fastqc.yml +++ b/.github/workflows/fastqc.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/gffread.yml b/.github/workflows/gffread.yml index 7509eafe..01ffa6c2 100644 --- a/.github/workflows/gffread.yml +++ b/.github/workflows/gffread.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/multiqc.yml b/.github/workflows/multiqc.yml index 4070a1df..79d68e94 100644 --- a/.github/workflows/multiqc.yml +++ b/.github/workflows/multiqc.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_collectmultiplemetrics.yml b/.github/workflows/picard_collectmultiplemetrics.yml index 86130f75..84723c8a 100644 --- a/.github/workflows/picard_collectmultiplemetrics.yml +++ b/.github/workflows/picard_collectmultiplemetrics.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_markduplicates.yml b/.github/workflows/picard_markduplicates.yml index 5438d30b..ff6d7330 100644 --- a/.github/workflows/picard_markduplicates.yml +++ b/.github/workflows/picard_markduplicates.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_mergesamfiles.yml b/.github/workflows/picard_mergesamfiles.yml index aa8fa1a4..92568d10 100644 --- a/.github/workflows/picard_mergesamfiles.yml +++ b/.github/workflows/picard_mergesamfiles.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/preseq_lcextrap.yml b/.github/workflows/preseq_lcextrap.yml index ea04bc8b..5dbb5369 100644 --- a/.github/workflows/preseq_lcextrap.yml +++ b/.github/workflows/preseq_lcextrap.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_flagstat.yml b/.github/workflows/samtools_flagstat.yml index 3b2a956b..4c9f8396 100644 --- a/.github/workflows/samtools_flagstat.yml +++ b/.github/workflows/samtools_flagstat.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_idxstats.yml b/.github/workflows/samtools_idxstats.yml index ffd4ce41..d3f544c7 100644 --- a/.github/workflows/samtools_idxstats.yml +++ b/.github/workflows/samtools_idxstats.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_index.yml b/.github/workflows/samtools_index.yml index 74ea3e45..4bfde947 100644 --- a/.github/workflows/samtools_index.yml +++ b/.github/workflows/samtools_index.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_sort.yml b/.github/workflows/samtools_sort.yml index 44b4bc12..45e7b563 100644 --- a/.github/workflows/samtools_sort.yml +++ b/.github/workflows/samtools_sort.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_stats.yml b/.github/workflows/samtools_stats.yml index ba346e60..e3333d96 100644 --- a/.github/workflows/samtools_stats.yml +++ b/.github/workflows/samtools_stats.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/stringtie.yml b/.github/workflows/stringtie.yml index d7bb6fff..6a9606a1 100644 --- a/.github/workflows/stringtie.yml +++ b/.github/workflows/stringtie.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/trimgalore.yml b/.github/workflows/trimgalore.yml index 5bc54de5..392ced96 100644 --- a/.github/workflows/trimgalore.yml +++ b/.github/workflows/trimgalore.yml @@ -45,5 +45,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/ucsc_bedgraphtobigwig.yml b/.github/workflows/ucsc_bedgraphtobigwig.yml index 935ad9aa..300df97c 100644 --- a/.github/workflows/ucsc_bedgraphtobigwig.yml +++ b/.github/workflows/ucsc_bedgraphtobigwig.yml @@ -44,5 +44,6 @@ jobs: with: name: logs path: | + /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out /tmp/pytest_workflow_*/*/log.err From 95270092b0995defd876a9fe302bffe58737185b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 13:32:54 -0600 Subject: [PATCH 04/22] ci: Add universal CI Test --- .github/workflows/pytest-workflow.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/pytest-workflow.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml new file mode 100644 index 00000000..538260eb --- /dev/null +++ b/.github/workflows/pytest-workflow.yml @@ -0,0 +1,19 @@ +name: Pytest-workflow +on: [push, pull_request] + +jobs: + ci_test: + runs-on: ubuntu-latest + name: CI Test + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + - name: Nextflow pytest-workflow + uses: Emiller88/pytest-workflow-composite-action@v0.1 + with: + pytest_tags: 'fastqc' + nxf_version: ${{ matrix.nxf_version }} From d00bd5f6b11041e4332668eb2bca28eeabc9c82f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 22:06:01 -0600 Subject: [PATCH 05/22] ci: v0.1 => master Living life on the edge --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 538260eb..eb3c14e3 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Nextflow pytest-workflow - uses: Emiller88/pytest-workflow-composite-action@v0.1 + uses: Emiller88/pytest-workflow-composite-action@master with: pytest_tags: 'fastqc' nxf_version: ${{ matrix.nxf_version }} From 474a6fad64033028d2ad131af18aa0f691834f8e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 22 Jan 2021 22:07:55 -0600 Subject: [PATCH 06/22] ci: Add python setup Cause apparently you can use an action inside of an action. Yet. --- .github/workflows/pytest-workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index eb3c14e3..2c7403c2 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -12,6 +12,10 @@ jobs: NXF_ANSI_LOG: false steps: - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" - name: Nextflow pytest-workflow uses: Emiller88/pytest-workflow-composite-action@master with: From f4316480c5f6ad9b71a0fe49a3d50d5fafb487a3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 22:55:07 -0600 Subject: [PATCH 07/22] ci: Use paths-filter to make one CI test --- .github/filters.yml | 4 ++++ .github/workflows/pytest-workflow.yml | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .github/filters.yml diff --git a/.github/filters.yml b/.github/filters.yml new file mode 100644 index 00000000..d65430e9 --- /dev/null +++ b/.github/filters.yml @@ -0,0 +1,4 @@ +fastqc: + - software/fastqc/** + - .github/workflows/fastqc.yml + - tests/software/fastqc/** diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 2c7403c2..348322cf 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -2,12 +2,26 @@ name: Pytest-workflow on: [push, pull_request] jobs: + changes: + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: '.github/filters.yml' + ci_test: runs-on: ubuntu-latest name: CI Test + needs: changes strategy: matrix: nxf_version: [20.11.0-edge] + tags: ${{ fromJson(needs.changes.outputs.modules) }} env: NXF_ANSI_LOG: false steps: @@ -19,5 +33,5 @@ jobs: - name: Nextflow pytest-workflow uses: Emiller88/pytest-workflow-composite-action@master with: - pytest_tags: 'fastqc' + pytest_tags: ${{ matrix.tags }} nxf_version: ${{ matrix.nxf_version }} From 3aaacdde09ad779ed8cb44806082e11be8932079 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 22:58:13 -0600 Subject: [PATCH 08/22] ci: Checkout code before looking for changes --- .github/workflows/pytest-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 348322cf..c36ed8a8 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -8,11 +8,11 @@ jobs: # Expose matched filters as job 'modules' output variable modules: ${{ steps.filter.outputs.changes }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: '.github/filters.yml' + - uses: actions/checkout@v2 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: '.github/filters.yml' ci_test: runs-on: ubuntu-latest From 939b1490b76531b2dc6bc3bc9c41c5c7e1e9b92a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:08:14 -0600 Subject: [PATCH 09/22] ci: Check if there's a change to anything --- .github/filters.yml | 3 +++ .github/workflows/pytest-workflow.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/filters.yml b/.github/filters.yml index d65430e9..fd862f50 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -1,3 +1,6 @@ +any: + - "**/*" + fastqc: - software/fastqc/** - .github/workflows/fastqc.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index c36ed8a8..1fce5387 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,6 +18,7 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes + if: steps.filter.outputs.any == 'true' strategy: matrix: nxf_version: [20.11.0-edge] From 9577f5dd646c8a3fe59fdaed57755fb2c18e7453 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:12:54 -0600 Subject: [PATCH 10/22] ci: Reference output rather than a step The step wasn't there --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 1fce5387..83ced0ce 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes - if: steps.filter.outputs.any == 'true' + if: ${{ fromJson(needs.changes.outputs.modules.any) }} == 'true' strategy: matrix: nxf_version: [20.11.0-edge] From 0f1016e4a30cf5e736b20414766409883c868843 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:25:53 -0600 Subject: [PATCH 11/22] ci: Remove any filter --- .github/filters.yml | 3 --- .github/workflows/pytest-workflow.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/filters.yml b/.github/filters.yml index fd862f50..d65430e9 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -1,6 +1,3 @@ -any: - - "**/*" - fastqc: - software/fastqc/** - .github/workflows/fastqc.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 83ced0ce..b6f29601 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes - if: ${{ fromJson(needs.changes.outputs.modules.any) }} == 'true' + if: ${{ fromJson(needs.changes.outputs.modules) }} == 'true' strategy: matrix: nxf_version: [20.11.0-edge] From 860a49d45624c0af95e77855764c517b43303206 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:35:44 -0600 Subject: [PATCH 12/22] ci: Need to wrap matrix's in [] --- .github/workflows/pytest-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index b6f29601..9ca8a7b4 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,11 +18,11 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes - if: ${{ fromJson(needs.changes.outputs.modules) }} == 'true' + if: ${{ fromJson(needs.changes.outputs.modules) }} != 'false' strategy: matrix: - nxf_version: [20.11.0-edge] - tags: ${{ fromJson(needs.changes.outputs.modules) }} + nxf_version: ['20.11.0-edge'] + tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] env: NXF_ANSI_LOG: false steps: From 13539e1c1a3bae45e4524d019fec73319740e1e4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:39:32 -0600 Subject: [PATCH 13/22] ci: Change name of pytest step --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 9ca8a7b4..dea17bac 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -14,7 +14,7 @@ jobs: with: filters: '.github/filters.yml' - ci_test: + test: runs-on: ubuntu-latest name: CI Test needs: changes From b008afe2e69d326d0f68c8987bc7ac8c2e8b3410 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:39:51 -0600 Subject: [PATCH 14/22] ci(#119): Upload logs on failure --- .github/workflows/pytest-workflow.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index dea17bac..415c20c6 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -36,3 +36,13 @@ jobs: with: pytest_tags: ${{ matrix.tags }} nxf_version: ${{ matrix.nxf_version }} + + - name: Upload logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + /tmp/pytest_workflow_*/*/.nextflow.log + /tmp/pytest_workflow_*/*/log.out + /tmp/pytest_workflow_*/*/log.err From 506ebe515ebf74441c35fa227f036f106762883c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 23 Jan 2021 23:50:20 -0600 Subject: [PATCH 15/22] ci: Move all files to filter file and delete old tests --- .github/filters.yml | 74 ++++++++++++++++++- .github/workflows/bowtie_align.yml | 52 ------------- .github/workflows/bowtie_index.yml | 50 ------------- .github/workflows/bwa_index.yml | 50 ------------- .github/workflows/bwa_mem.yml | 50 ------------- .github/workflows/fastqc.yml | 50 ------------- .github/workflows/gffread.yml | 50 ------------- .github/workflows/multiqc.yml | 50 ------------- .../picard_collectmultiplemetrics.yml | 50 ------------- .github/workflows/picard_markduplicates.yml | 50 ------------- .github/workflows/picard_mergesamfiles.yml | 50 ------------- .github/workflows/preseq_lcextrap.yml | 50 ------------- .github/workflows/samtools_flagstat.yml | 50 ------------- .github/workflows/samtools_idxstats.yml | 50 ------------- .github/workflows/samtools_index.yml | 50 ------------- .github/workflows/samtools_sort.yml | 50 ------------- .github/workflows/samtools_stats.yml | 50 ------------- .github/workflows/stringtie.yml | 50 ------------- .github/workflows/trimgalore.yml | 50 ------------- .github/workflows/ucsc_bedgraphtobigwig.yml | 49 ------------ 20 files changed, 73 insertions(+), 952 deletions(-) delete mode 100644 .github/workflows/bowtie_align.yml delete mode 100644 .github/workflows/bowtie_index.yml delete mode 100644 .github/workflows/bwa_index.yml delete mode 100644 .github/workflows/bwa_mem.yml delete mode 100644 .github/workflows/fastqc.yml delete mode 100644 .github/workflows/gffread.yml delete mode 100644 .github/workflows/multiqc.yml delete mode 100644 .github/workflows/picard_collectmultiplemetrics.yml delete mode 100644 .github/workflows/picard_markduplicates.yml delete mode 100644 .github/workflows/picard_mergesamfiles.yml delete mode 100644 .github/workflows/preseq_lcextrap.yml delete mode 100644 .github/workflows/samtools_flagstat.yml delete mode 100644 .github/workflows/samtools_idxstats.yml delete mode 100644 .github/workflows/samtools_index.yml delete mode 100644 .github/workflows/samtools_sort.yml delete mode 100644 .github/workflows/samtools_stats.yml delete mode 100644 .github/workflows/stringtie.yml delete mode 100644 .github/workflows/trimgalore.yml delete mode 100644 .github/workflows/ucsc_bedgraphtobigwig.yml diff --git a/.github/filters.yml b/.github/filters.yml index d65430e9..03e21d09 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -1,4 +1,76 @@ +bowtie_align: + - software/bowtie/align/** + - software/bowtie/index/** + - tests/software/bowtie/** + +bowtie_index: + - software/bowtie/index/** + - tests/software/bowtie/** + +bwa_index: + - software/bwa/index/** + - tests/software/bwa/** + +bwa_mem: + - software/bwa/mem/** + - tests/software/bwa/** + fastqc: - software/fastqc/** - - .github/workflows/fastqc.yml - tests/software/fastqc/** + +gffread: + - software/gffread/** + - tests/software/gffread/** + +multiqc: + - software/multiqc/** + - tests/software/multiqc/** + +picard_collectmultiplemetrics: + - software/picard/collectmultiplemetrics/** + - tests/software/picard/** + +picard_markduplicates: + - software/picard/markduplicates/** + - tests/software/picard/** + +picard_mergesamfiles: + - software/picard/mergesamfiles/** + - tests/software/picard/** + +preseq_lcextrap: + - software/preseq/lcextrap/** + - tests/software/preseq/** + +samtools_flagstat: + - software/samtools/flagstat/** + - tests/software/samtools/** + +samtools_idxstats: + - software/samtools/idxstats/** + - tests/software/samtools/** + +samtools_index: + - software/samtools/index/** + - tests/software/samtools/** + +samtools_sort: + - software/samtools/sort/** + - tests/software/samtools/** + +samtools_stats: + - software/samtools/stats/** + - tests/software/samtools/** + +stringtie: + - software/stringtie/** + - tests/software/stringtie/** + +trimgalore: + - software/trimgalore/** + - tests/software/trimgalore/** + +ucsc_bedgraphtobigwig: + - software/ucsc/bedgraphtobigwig/** + - tests/software/ucsc/** diff --git a/.github/workflows/bowtie_align.yml b/.github/workflows/bowtie_align.yml deleted file mode 100644 index a5ac590c..00000000 --- a/.github/workflows/bowtie_align.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: bowtie_align -on: - push: - paths: - - software/bowtie/align/** - - software/bowtie/index/** - - .github/workflows/bowtie_align.yml - - tests/software/bowtie/** - pull_request: - paths: - - software/bowtie/align/** - - software/bowtie/index/** - - .github/workflows/bowtie_align.yml - - tests/software/bowtie/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag bowtie_align --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bowtie_index.yml b/.github/workflows/bowtie_index.yml deleted file mode 100644 index cd1d7501..00000000 --- a/.github/workflows/bowtie_index.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: bowtie_index -on: - push: - paths: - - software/bowtie/index/** - - .github/workflows/bowtie_index.yml - - tests/software/bowtie/** - pull_request: - paths: - - software/bowtie/index/** - - .github/workflows/bowtie_index.yml - - tests/software/bowtie/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag bowtie_index --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_index.yml b/.github/workflows/bwa_index.yml deleted file mode 100644 index 632dc5be..00000000 --- a/.github/workflows/bwa_index.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: bwa_index -on: - push: - paths: - - software/bwa/index/** - - .github/workflows/software/bwa_index.yml - - tests/software/bwa/** - pull_request: - paths: - - software/bwa/index/** - - .github/workflows/software/bwa_index.yml - - tests/software/bwa/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag bwa_index --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/bwa_mem.yml b/.github/workflows/bwa_mem.yml deleted file mode 100644 index 2c5ccbcc..00000000 --- a/.github/workflows/bwa_mem.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: bwa_mem -on: - push: - paths: - - software/bwa/mem/** - - .github/workflows/bwa_mem.yml - - tests/software/bwa/** - pull_request: - paths: - - software/bwa/mem/** - - .github/workflows/bwa_mem.yml - - tests/software/bwa/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag bwa_index --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/fastqc.yml b/.github/workflows/fastqc.yml deleted file mode 100644 index e9fae495..00000000 --- a/.github/workflows/fastqc.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: fastqc -on: - push: - paths: - - software/fastqc/** - - .github/workflows/fastqc.yml - - tests/software/fastqc/** - pull_request: - paths: - - software/fastqc/** - - .github/workflows/fastqc.yml - - tests/software/fastqc/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag fastqc --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/gffread.yml b/.github/workflows/gffread.yml deleted file mode 100644 index 01ffa6c2..00000000 --- a/.github/workflows/gffread.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: gffread -on: - push: - paths: - - software/gffread/** - - .github/workflows/gffread.yml - - tests/software/gffread/** - pull_request: - paths: - - software/gffread/** - - .github/workflows/gffread.yml - - tests/software/gffread/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag gffread --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/multiqc.yml b/.github/workflows/multiqc.yml deleted file mode 100644 index 79d68e94..00000000 --- a/.github/workflows/multiqc.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: multiqc -on: - push: - paths: - - software/multiqc/** - - .github/workflows/multiqc.yml - - tests - pull_request: - paths: - - software/multiqc/** - - .github/workflows/multiqc.yml - - tests - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag multiqc --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_collectmultiplemetrics.yml b/.github/workflows/picard_collectmultiplemetrics.yml deleted file mode 100644 index 84723c8a..00000000 --- a/.github/workflows/picard_collectmultiplemetrics.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: picard_collectmultiplemetrics -on: - push: - paths: - - software/picard/collectmultiplemetrics/** - - .github/workflows/picard_collectmultiplemetrics.yml - - tests/software/picard/** - pull_request: - paths: - - software/picard/collectmultiplemetrics/** - - .github/workflows/picard_collectmultiplemetrics.yml - - tests/software/picard/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag picard_collectmultiplemetrics --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_markduplicates.yml b/.github/workflows/picard_markduplicates.yml deleted file mode 100644 index ff6d7330..00000000 --- a/.github/workflows/picard_markduplicates.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: picard_markduplicates -on: - push: - paths: - - software/picard/markduplicates/** - - .github/workflows/picard_markduplicates.yml - - tests/software/picard/** - pull_request: - paths: - - software/picard/markduplicates/** - - .github/workflows/picard_markduplicates.yml - - tests/software/picard/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag picard_markduplicates --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/picard_mergesamfiles.yml b/.github/workflows/picard_mergesamfiles.yml deleted file mode 100644 index 92568d10..00000000 --- a/.github/workflows/picard_mergesamfiles.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: picard_mergesamfiles -on: - push: - paths: - - software/picard/mergesamfiles/** - - .github/workflows/picard_mergesamfiles.yml - - tests/software/picard/** - pull_request: - paths: - - software/picard/mergesamfiles/** - - .github/workflows/picard_mergesamfiles.yml - - tests/software/picard/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag picard_mergesamfiles --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/preseq_lcextrap.yml b/.github/workflows/preseq_lcextrap.yml deleted file mode 100644 index 5dbb5369..00000000 --- a/.github/workflows/preseq_lcextrap.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: preseq_lcextrap -on: - push: - paths: - - software/preseq/lcextrap/** - - .github/workflows/preseq_lcextrap.yml - - tests/software/preseq/** - pull_request: - paths: - - software/preseq/lcextrap/** - - .github/workflows/preseq_lcextrap.yml - - tests/software/preseq/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag preseq_lcextrap --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_flagstat.yml b/.github/workflows/samtools_flagstat.yml deleted file mode 100644 index 4c9f8396..00000000 --- a/.github/workflows/samtools_flagstat.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: samtools_flagstat -on: - push: - paths: - - software/samtools/flagstat/** - - .github/workflows/samtools_flagstat.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/flagstat/** - - .github/workflows/samtools_flagstat.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag samtools_flagstat --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_idxstats.yml b/.github/workflows/samtools_idxstats.yml deleted file mode 100644 index d3f544c7..00000000 --- a/.github/workflows/samtools_idxstats.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: samtools_idxstats -on: - push: - paths: - - software/samtools/idxstats/** - - .github/workflows/samtools_idxstats.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/idxstats/** - - .github/workflows/samtools_idxstats.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag samtools_idxstats --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_index.yml b/.github/workflows/samtools_index.yml deleted file mode 100644 index 4bfde947..00000000 --- a/.github/workflows/samtools_index.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: samtools_index -on: - push: - paths: - - software/samtools/index/** - - .github/workflows/samtools_index.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/index/** - - .github/workflows/samtools_index.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag samtools_index --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_sort.yml b/.github/workflows/samtools_sort.yml deleted file mode 100644 index 45e7b563..00000000 --- a/.github/workflows/samtools_sort.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: samtools_sort -on: - push: - paths: - - software/samtools/sort/** - - .github/workflows/samtools_sort.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/sort/** - - .github/workflows/samtools_sort.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag samtools_sort --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/samtools_stats.yml b/.github/workflows/samtools_stats.yml deleted file mode 100644 index e3333d96..00000000 --- a/.github/workflows/samtools_stats.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: samtools_stats -on: - push: - paths: - - software/samtools/stats/** - - .github/workflows/software/samtools_stats.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/stats/** - - .github/workflows/software/samtools_stats.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag samtools_stats --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/stringtie.yml b/.github/workflows/stringtie.yml deleted file mode 100644 index 6a9606a1..00000000 --- a/.github/workflows/stringtie.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: stringtie -on: - push: - paths: - - software/stringtie/** - - .github/workflows/stringtie.yml - - tests/software/stringtie/** - pull_request: - paths: - - software/stringtie/** - - .github/workflows/stringtie.yml - - tests/software/stringtie/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag stringtie --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/trimgalore.yml b/.github/workflows/trimgalore.yml deleted file mode 100644 index 392ced96..00000000 --- a/.github/workflows/trimgalore.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: trimgalore -on: - push: - paths: - - software/trimgalore/** - - .github/workflows/trimgalore.yml - - tests/software/trimgalore/** - pull_request: - paths: - - software/trimgalore/** - - .github/workflows/trimgalore.yml - - tests/software/trimgalore/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag trimgalore --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err diff --git a/.github/workflows/ucsc_bedgraphtobigwig.yml b/.github/workflows/ucsc_bedgraphtobigwig.yml deleted file mode 100644 index 300df97c..00000000 --- a/.github/workflows/ucsc_bedgraphtobigwig.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: ucsc_bedgraphtobigwig -on: - push: - paths: - - software/ucsc/bedgraphtobigwig/** - - .github/workflows/ucsc_bedgraphtobigwig.yml - - tests/software/ucsc/** - pull_request: - paths: - - software/ucsc/bedgraphtobigwig/** - - .github/workflows/ucsc_bedgraphtobigwig.yml - - tests/software/ucsc/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --kwdof --tag ucsc_bedgraphtobigwig --symlink --wt 2 - - - name: Upload logs on failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: logs - path: | - /tmp/pytest_workflow_*/*/.nextflow.log - /tmp/pytest_workflow_*/*/log.out - /tmp/pytest_workflow_*/*/log.err From 7a89902657c68317a726a24d7820926ff31d2bdd Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 10:20:00 -0600 Subject: [PATCH 16/22] ci: Fix check for no checks https://github.community/t/needs-job-outputs-issue/149156 https://github.community/t/jobs-being-skipped-while-using-both-needs-and-if-unclear-why/18480 --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 415c20c6..c53602c9 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes - if: ${{ fromJson(needs.changes.outputs.modules) }} != 'false' + if: needs.changes.outputs.modules == 'true' strategy: matrix: nxf_version: ['20.11.0-edge'] From cff3ac7e5f74dc6574dec90c778e2f65cbc4d755 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 10:37:29 -0600 Subject: [PATCH 17/22] ci: Make an edit to test pipeline trigger Plus this got clean up in 0c9f29e0a2e7f1ed84b5ca854467316fe9d056ce --- tests/software/fastqc/main.nf | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/software/fastqc/main.nf b/tests/software/fastqc/main.nf index 2a4b8677..27411a17 100644 --- a/tests/software/fastqc/main.nf +++ b/tests/software/fastqc/main.nf @@ -34,5 +34,3 @@ workflow test_paired_end { zip = FASTQC_PE.out.zip } - -// TODO Test e2e From b048bf33e0d730d905fd4ee8c831d5f40ae85e60 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 11:14:48 -0600 Subject: [PATCH 18/22] ci: Check to see if it's an example array --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index c53602c9..ee294e30 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest name: CI Test needs: changes - if: needs.changes.outputs.modules == 'true' + if: needs.changes.outputs.modules != '[]' strategy: matrix: nxf_version: ['20.11.0-edge'] From 73b572eede026d60a5bc23bcaf05c512d5d72d6d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 11:25:13 -0600 Subject: [PATCH 19/22] ci: Cache pip --- .github/workflows/pytest-workflow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index ee294e30..c4bc0f02 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -27,6 +27,12 @@ jobs: NXF_ANSI_LOG: false steps: - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Set up Python uses: actions/setup-python@v2 with: From c7262e27d867bd2ea1ac6210d34e9cf80a84d20c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 11:35:24 -0600 Subject: [PATCH 20/22] ci: Cache nextflow --- .github/workflows/pytest-workflow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index c4bc0f02..5a55c651 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -37,6 +37,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.x" + - uses: actions/cache@v2 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }} + restore-keys: | + ${{ runner.os }}-nextflow- - name: Nextflow pytest-workflow uses: Emiller88/pytest-workflow-composite-action@master with: From 6da00a8a1b4bf8d4eae5d6e268caa4b744df912b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 24 Jan 2021 11:38:29 -0600 Subject: [PATCH 21/22] ci: Put python setup before cache https://github.com/backstage/backstage/commit/34cf8a3e0f38def29a0bfc13ac498d9ed9c955a0 --- .github/workflows/pytest-workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 5a55c651..8cc9b9f5 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -27,16 +27,16 @@ jobs: NXF_ANSI_LOG: false steps: - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" - uses: actions/cache@v2 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - uses: actions/cache@v2 with: path: /usr/local/bin/nextflow From ce2a5b32f5bb59b4a8c3d74d7b6df5a9f2251832 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 12:18:59 -0600 Subject: [PATCH 22/22] refactor: Move away from composite-action --- .github/workflows/pytest-workflow.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 8cc9b9f5..2e2edc69 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -3,6 +3,7 @@ on: [push, pull_request] jobs: changes: + name: Check for changes runs-on: ubuntu-latest outputs: # Expose matched filters as job 'modules' output variable @@ -43,11 +44,24 @@ jobs: key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }} restore-keys: | ${{ runner.os }}-nextflow- - - name: Nextflow pytest-workflow - uses: Emiller88/pytest-workflow-composite-action@master + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 with: - pytest_tags: ${{ matrix.tags }} - nxf_version: ${{ matrix.nxf_version }} + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - name: Run pytest-workflow + run: pytest --tag ${{ matrix.tags }} --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }}