From ddbca7b467b8f84b05d376bfcf7ff2c32e2588a5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 11:55:41 -0600 Subject: [PATCH 01/20] test: Use profile ENV variable hack The user will now set the env variable and pass it to pytest. You also can't set the env variable ahead of time, it has to be run with pytest. Example: PROFILE=conda pytest --tag fastqc --symlink --kwd --- tests/config/nextflow.config | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 4149feea..9af14ccf 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -1,20 +1,15 @@ - params { outdir = "output/" publish_dir_mode = "copy" enable_conda = false } -profiles { - conda { - params.enable_conda = true - } - docker { - docker.enabled = true - docker.runOptions = '-u \$(id -u):\$(id -g)' - } - singularity { - singularity.enabled = true - singularity.autoMounts = true - } +if("$PROFILE" == "docker") { + docker.enabled = true + docker.runOptions = '-u \$(id -u):\$(id -g)' +} else if ("$PROFILE" == "singularity") { + singularity.enabled = true + singularity.autoMounts = true +} else if ("$PROFILE" == "conda") { + params.enable_conda = true } From 341fa43ae322876b10526cabb2ca3b2175fdaad6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 12:00:13 -0600 Subject: [PATCH 02/20] ci: Add PROFILE ahead of pytest run --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 2e2edc69..b3775870 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -61,7 +61,7 @@ jobs: # Test the module - name: Run pytest-workflow - run: pytest --tag ${{ matrix.tags }} --symlink --wt 2 + run: PROFILE=docker pytest --tag ${{ matrix.tags }} --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} From 35ac5c5f1504b697a9ee3ae5cae730bcc14ec1bb Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:27:45 -0600 Subject: [PATCH 03/20] ci: Add filters for new modules These snuck in after the 506ebe5 was written --- .github/filters.yml | 68 ++++++++++++++++++++++- .github/workflows/bowtie2_align.yml | 42 -------------- .github/workflows/bowtie2_build.yml | 40 ------------- .github/workflows/bowtie_build.yml | 40 ------------- .github/workflows/cutadapt.yml | 40 ------------- .github/workflows/dsh_filterbed.yml | 40 ------------- .github/workflows/dsh_splitbed.yml | 40 ------------- .github/workflows/fastp.yml | 40 ------------- .github/workflows/pangolin.yml | 40 ------------- .github/workflows/quast.yml | 40 ------------- .github/workflows/salmon_index.yml | 40 ------------- .github/workflows/salmon_quant.yml | 40 ------------- .github/workflows/samtools_mpileup.yml | 40 ------------- .github/workflows/samtools_view.yml | 40 ------------- .github/workflows/seacr_callpeak.yml | 40 ------------- .github/workflows/star_align.yml | 40 ------------- .github/workflows/star_genomegenerate.yml | 40 ------------- 17 files changed, 65 insertions(+), 645 deletions(-) delete mode 100644 .github/workflows/bowtie2_align.yml delete mode 100644 .github/workflows/bowtie2_build.yml delete mode 100644 .github/workflows/bowtie_build.yml delete mode 100644 .github/workflows/cutadapt.yml delete mode 100644 .github/workflows/dsh_filterbed.yml delete mode 100644 .github/workflows/dsh_splitbed.yml delete mode 100644 .github/workflows/fastp.yml delete mode 100644 .github/workflows/pangolin.yml delete mode 100644 .github/workflows/quast.yml delete mode 100644 .github/workflows/salmon_index.yml delete mode 100644 .github/workflows/salmon_quant.yml delete mode 100644 .github/workflows/samtools_mpileup.yml delete mode 100644 .github/workflows/samtools_view.yml delete mode 100644 .github/workflows/seacr_callpeak.yml delete mode 100644 .github/workflows/star_align.yml delete mode 100644 .github/workflows/star_genomegenerate.yml diff --git a/.github/filters.yml b/.github/filters.yml index 03e21d09..73f4637a 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -1,10 +1,19 @@ +bowtie2_align: + - software/bowtie2/align/** + - software/bowtie2/build/** + - tests/software/bowtie2/** + +bowtie2_build: + - software/bowtie2/build/** + - tests/software/bowtie2/** + bowtie_align: - software/bowtie/align/** - - software/bowtie/index/** + - software/bowtie/build/** - tests/software/bowtie/** -bowtie_index: - - software/bowtie/index/** +bowtie_build: + - software/bowtie/build/** - tests/software/bowtie/** bwa_index: @@ -15,6 +24,22 @@ bwa_mem: - software/bwa/mem/** - tests/software/bwa/** +cutadapt: + - software/cutadapt/** + - tests/software/cutadapt/** + +dsh_filterbed: + - software/dsh/filterbed/** + - tests/software/dsh/** + +dsh_splitbed: + - software/dsh/splitbed/** + - tests/software/dsh/** + +fastp: + - software/fastp/** + - tests/software/fastp/** + fastqc: - software/fastqc/** - tests/software/fastqc/** @@ -27,6 +52,10 @@ multiqc: - software/multiqc/** - tests/software/multiqc/** +pangolin: + - software/pangolin/** + - tests/software/pangolin/** + picard_collectmultiplemetrics: - software/picard/collectmultiplemetrics/** - tests/software/picard/** @@ -43,6 +72,18 @@ preseq_lcextrap: - software/preseq/lcextrap/** - tests/software/preseq/** +quast: + - software/quast/** + - tests/software/quast/** + +salmon_index: + - software/salmon/index/** + - tests/software/salmon/** + +salmon_quant: + - software/salmon/quant/** + - tests/software/salmon/** + samtools_flagstat: - software/samtools/flagstat/** - tests/software/samtools/** @@ -55,6 +96,10 @@ samtools_index: - software/samtools/index/** - tests/software/samtools/** +samtools_mpileup: + - software/samtools/mpileup/** + - tests/software/samtools/** + samtools_sort: - software/samtools/sort/** - tests/software/samtools/** @@ -63,6 +108,23 @@ samtools_stats: - software/samtools/stats/** - tests/software/samtools/** +samtools_view: + - software/samtools/view/** + - tests/software/samtools/** + +seacr_callpeak: + - software/seacr/callpeak/** + - tests/software/seacr/** + +star_align: + - software/star/align/** + - tests/software/star/** + +star_genomegenerate: + - software/star/genomegenerate/** + - tests/software/star/** + + stringtie: - software/stringtie/** - tests/software/stringtie/** diff --git a/.github/workflows/bowtie2_align.yml b/.github/workflows/bowtie2_align.yml deleted file mode 100644 index 3642b5c9..00000000 --- a/.github/workflows/bowtie2_align.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: bowtie2_align -on: - push: - paths: - - software/bowtie2/align/** - - software/bowtie2/build/** - - .github/workflows/bowtie2_align.yml - - tests/software/bowtie2/** - pull_request: - paths: - - software/bowtie2/align/** - - software/bowtie2/build/** - - .github/workflows/bowtie2_align.yml - - tests/software/bowtie2/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bowtie2_align --symlink --wt 2 diff --git a/.github/workflows/bowtie2_build.yml b/.github/workflows/bowtie2_build.yml deleted file mode 100644 index 21ddde78..00000000 --- a/.github/workflows/bowtie2_build.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: bowtie2_build -on: - push: - paths: - - software/bowtie2/build/** - - .github/workflows/bowtie2_build.yml - - tests/software/bowtie2/** - pull_request: - paths: - - software/bowtie2/build/** - - .github/workflows/bowtie2_build.yml - - tests/software/bowtie2/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bowtie2_build --symlink --wt 2 diff --git a/.github/workflows/bowtie_build.yml b/.github/workflows/bowtie_build.yml deleted file mode 100644 index 1a02341e..00000000 --- a/.github/workflows/bowtie_build.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: bowtie_build -on: - push: - paths: - - software/bowtie/build/** - - .github/workflows/bowtie_build.yml - - tests/software/bowtie/** - pull_request: - paths: - - software/bowtie/build/** - - .github/workflows/bowtie_build.yml - - tests/software/bowtie/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bowtie_build --symlink --wt 2 diff --git a/.github/workflows/cutadapt.yml b/.github/workflows/cutadapt.yml deleted file mode 100644 index 22bc5171..00000000 --- a/.github/workflows/cutadapt.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: cutadapt -on: - push: - paths: - - software/cutadapt/** - - .github/workflows/cutadapt.yml - - tests/software/cutadapt/** - pull_request: - paths: - - software/cutadapt/** - - .github/workflows/cutadapt.yml - - tests/software/cutadapt/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag cutadapt --symlink --wt 2 diff --git a/.github/workflows/dsh_filterbed.yml b/.github/workflows/dsh_filterbed.yml deleted file mode 100644 index 1d99dbb8..00000000 --- a/.github/workflows/dsh_filterbed.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: dsh_filterbed -on: - push: - paths: - - software/dsh/filterbed/** - - .github/workflows/dsh_filterbed.yml - - tests/software/dsh/** - pull_request: - paths: - - software/dsh/filterbed/** - - .github/workflows/dsh_filterbed.yml - - tests/software/dsh/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag dsh_filterbed --symlink --wt 2 diff --git a/.github/workflows/dsh_splitbed.yml b/.github/workflows/dsh_splitbed.yml deleted file mode 100644 index f2ebbd88..00000000 --- a/.github/workflows/dsh_splitbed.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: dsh_splitbed -on: - push: - paths: - - software/dsh/splitbed/** - - .github/workflows/dsh_splitbed.yml - - tests/software/dsh/** - pull_request: - paths: - - software/dsh/splitbed/** - - .github/workflows/dsh_splitbed.yml - - tests/software/dsh/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag dsh_splitbed --symlink --wt 2 diff --git a/.github/workflows/fastp.yml b/.github/workflows/fastp.yml deleted file mode 100644 index 90ad15ef..00000000 --- a/.github/workflows/fastp.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: fastp -on: - push: - paths: - - software/fastp/** - - .github/workflows/fastp.yml - - tests - pull_request: - paths: - - software/fastp/** - - .github/workflows/fastp.yml - - tests - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag fastp --symlink --wt 2 diff --git a/.github/workflows/pangolin.yml b/.github/workflows/pangolin.yml deleted file mode 100644 index 44d718f7..00000000 --- a/.github/workflows/pangolin.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: pangolin -on: - push: - paths: - - software/pangolin/** - - .github/workflows/pangolin.yml - - tests - pull_request: - paths: - - software/pangolin/** - - .github/workflows/pangolin.yml - - tests - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag pangolin --symlink --wt 2 diff --git a/.github/workflows/quast.yml b/.github/workflows/quast.yml deleted file mode 100644 index 669d916b..00000000 --- a/.github/workflows/quast.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: quast -on: - push: - paths: - - software/quast/** - - .github/workflows/quast.yml - - tests - pull_request: - paths: - - software/pquast/** - - .github/workflows/quest.yml - - tests - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag quast --symlink --wt 2 diff --git a/.github/workflows/salmon_index.yml b/.github/workflows/salmon_index.yml deleted file mode 100644 index e3b065e8..00000000 --- a/.github/workflows/salmon_index.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: salmon_index -on: - push: - paths: - - software/salmon/index/** - - .github/workflows/salmon_index.yml - - tests/software/salmon/** - pull_request: - paths: - - software/salmon/index/** - - .github/workflows/salmon_index.yml - - tests/software/salmon/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag salmon_index --symlink --wt 2 diff --git a/.github/workflows/salmon_quant.yml b/.github/workflows/salmon_quant.yml deleted file mode 100644 index 16a6bf9c..00000000 --- a/.github/workflows/salmon_quant.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: salmon_quant -on: - push: - paths: - - software/salmon/quant/** - - .github/workflows/salmon_quant.yml - - tests/software/salmon/** - pull_request: - paths: - - software/salmon/quant/** - - .github/workflows/salmon_quant.yml - - tests/software/salmon/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag salmon_quant --symlink --wt 2 diff --git a/.github/workflows/samtools_mpileup.yml b/.github/workflows/samtools_mpileup.yml deleted file mode 100644 index 76d92996..00000000 --- a/.github/workflows/samtools_mpileup.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: samtools_mpileup -on: - push: - paths: - - software/samtools/mpileup/** - - .github/workflows/samtools_mpileup.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/mpileup/** - - .github/workflows/samtools_mpileup.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag samtools_mpileup --symlink --wt 2 diff --git a/.github/workflows/samtools_view.yml b/.github/workflows/samtools_view.yml deleted file mode 100644 index a73ef9e3..00000000 --- a/.github/workflows/samtools_view.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: samtools_view -on: - push: - paths: - - software/samtools/view/** - - .github/workflows/samtools_view.yml - - tests/software/samtools/** - pull_request: - paths: - - software/samtools/view/** - - .github/workflows/samtools_view.yml - - tests/software/samtools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag samtools_view --symlink --wt 2 diff --git a/.github/workflows/seacr_callpeak.yml b/.github/workflows/seacr_callpeak.yml deleted file mode 100644 index 3d73a43e..00000000 --- a/.github/workflows/seacr_callpeak.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: seacr_callpeak -on: - push: - paths: - - software/seacr/callpeak/** - - .github/workflows/seacr_callpeak.yml - - tests/software/seacr/** - pull_request: - paths: - - software/seacr/callpeak/** - - .github/workflows/seacr_callpeak.yml - - tests/software/seacr/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag seacr_callpeak --symlink --wt 2 diff --git a/.github/workflows/star_align.yml b/.github/workflows/star_align.yml deleted file mode 100644 index 76dd333d..00000000 --- a/.github/workflows/star_align.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: star_genomegenerate -on: - push: - paths: - - software/star/align/** - - .github/workflows/star_align.yml - - tests/software/star/** - pull_request: - paths: - - software/star/align/** - - .github/workflows/star_align.yml - - tests/software/star/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag star_align --symlink --wt 2 diff --git a/.github/workflows/star_genomegenerate.yml b/.github/workflows/star_genomegenerate.yml deleted file mode 100644 index dcd3884e..00000000 --- a/.github/workflows/star_genomegenerate.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: star_genomegenerate -on: - push: - paths: - - software/star/genomegenerate/** - - .github/workflows/star_genomegenerate.yml - - tests/software/star/** - pull_request: - paths: - - software/star/genomegenerate/** - - .github/workflows/star_genomegenerate.yml - - tests/software/star/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag star_genomegenerate --symlink --wt 2 From 688cc05fe86d1bef99ccf8e58946d0ccf846a6d9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:30:59 -0600 Subject: [PATCH 04/20] tests: Remove -profile in tests See ddbca7b467b8f84b05d376bfcf7ff2c32e2588a5 for more info --- tests/software/bowtie/test.yml | 6 +++--- tests/software/bowtie2/test.yml | 6 +++--- tests/software/bwa/test.yml | 6 +++--- tests/software/cutadapt/test.yml | 4 ++-- tests/software/dsh/test.yml | 4 ++-- tests/software/fastp/test.yml | 4 ++-- tests/software/fastqc/test.yml | 4 ++-- tests/software/gffread/test.yml | 2 +- tests/software/multiqc/test.yml | 2 +- tests/software/pangolin/test.yml | 2 +- tests/software/picard/test.yml | 8 ++++---- tests/software/preseq/test.yml | 4 ++-- tests/software/quast/test.yml | 4 ++-- tests/software/salmon/test.yml | 6 +++--- tests/software/samtools/test.yml | 14 +++++++------- tests/software/seacr/test.yml | 2 +- tests/software/star/test.yml | 6 +++--- tests/software/stringtie/test.yml | 4 ++-- tests/software/trimgalore/test.yml | 4 ++-- tests/software/ucsc/test.yml | 2 +- 20 files changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/software/bowtie/test.yml b/tests/software/bowtie/test.yml index 96037548..61ed5f1d 100644 --- a/tests/software/bowtie/test.yml +++ b/tests/software/bowtie/test.yml @@ -1,5 +1,5 @@ - name: Run bowtie build - command: nextflow run ./tests/software/bowtie -profile docker -entry test_bowtie_build -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie -entry test_bowtie_build -c tests/config/nextflow.config tags: - bowtie - bowtie_build @@ -18,7 +18,7 @@ md5sum: f3c398bba5158f4039334a932d79c051 - name: Run bowtie build and align single-end - command: nextflow run ./tests/software/bowtie -profile docker -entry test_bowtie_alignment_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie -entry test_bowtie_alignment_single_end -c tests/config/nextflow.config tags: - bowtie - bowtie_align @@ -41,7 +41,7 @@ md5sum: 9feed8a55d4b5e600dcc577768ef07fc - name: Run bowtie index and align paired-end - command: nextflow run ./tests/software/bowtie -profile docker -entry test_bowtie_alignment_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie -entry test_bowtie_alignment_paired_end -c tests/config/nextflow.config tags: - bowtie - bowtie_align diff --git a/tests/software/bowtie2/test.yml b/tests/software/bowtie2/test.yml index a3c6f31d..f5ecbf12 100644 --- a/tests/software/bowtie2/test.yml +++ b/tests/software/bowtie2/test.yml @@ -1,5 +1,5 @@ - name: Run bowtie2 build - command: nextflow run ./tests/software/bowtie2 -profile docker -entry test_bowtie2_build -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie2 -entry test_bowtie2_build -c tests/config/nextflow.config tags: - bowtie2 - bowtie2_build @@ -18,7 +18,7 @@ md5sum: 5e8fb4af677eb3a57a40e76dc3f6db72 - name: Run bowtie2 index and align single-end - command: nextflow run ./tests/software/bowtie2 -profile docker -entry test_bowtie2_alignment_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie2 -entry test_bowtie2_alignment_single_end -c tests/config/nextflow.config tags: - bowtie2 - bowtie2_align @@ -41,7 +41,7 @@ md5sum: 906102f401d8234b6473790988fb09cf - name: Run bowtie2 index and align paired-end - command: nextflow run ./tests/software/bowtie2 -profile docker -entry test_bowtie2_alignment_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bowtie2 -entry test_bowtie2_alignment_paired_end -c tests/config/nextflow.config tags: - bowtie2 - bowtie2_align diff --git a/tests/software/bwa/test.yml b/tests/software/bwa/test.yml index 9efefa1d..3258af5a 100644 --- a/tests/software/bwa/test.yml +++ b/tests/software/bwa/test.yml @@ -1,5 +1,5 @@ - name: Run bwa index - command: nextflow run ./tests/software/bwa -profile docker -entry test_bwa_index -c tests/config/nextflow.config + command: nextflow run ./tests/software/bwa -entry test_bwa_index -c tests/config/nextflow.config tags: - bwa - bwa_index @@ -14,7 +14,7 @@ md5sum: a47dcc92e750e2f16fbd979b8ff9538e - name: Run bwa mem single-end - command: nextflow run ./tests/software/bwa -profile docker -entry test_bwa_mem_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bwa -entry test_bwa_mem_single_end -c tests/config/nextflow.config tags: - bwa - bwa_mem @@ -23,7 +23,7 @@ md5sum: 3ee21210bac387e0335008146e4728bc - name: Run bwa mem paired-end - command: nextflow run ./tests/software/bwa -profile docker -entry test_bwa_mem_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/bwa -entry test_bwa_mem_paired_end -c tests/config/nextflow.config tags: - bwa - bwa_mem diff --git a/tests/software/cutadapt/test.yml b/tests/software/cutadapt/test.yml index 39a06bc1..c91a0713 100644 --- a/tests/software/cutadapt/test.yml +++ b/tests/software/cutadapt/test.yml @@ -1,5 +1,5 @@ - name: Run cutadapt single-end test workflow - command: nextflow run ./tests/software/cutadapt -profile docker -entry test_cutadapt_se -c tests/config/nextflow.config + command: nextflow run ./tests/software/cutadapt -entry test_cutadapt_se -c tests/config/nextflow.config tags: - cutadapt - cutadapt_se @@ -8,7 +8,7 @@ - path: ./output/cutadapt/test.trim.fastq.gz - name: Run cutadapt paired-end test workflow - command: nextflow run ./tests/software/cutadapt -profile docker -entry test_cutadapt_pe -c tests/config/nextflow.config + command: nextflow run ./tests/software/cutadapt -entry test_cutadapt_pe -c tests/config/nextflow.config tags: - cutadapt - cutadapt_pe diff --git a/tests/software/dsh/test.yml b/tests/software/dsh/test.yml index 3e4261b7..f58d2065 100644 --- a/tests/software/dsh/test.yml +++ b/tests/software/dsh/test.yml @@ -1,5 +1,5 @@ - name: Run dsh filterbed - command: nextflow run ./tests/software/dsh -profile docker -entry test_dsh_filterbed -c tests/config/nextflow.config + command: nextflow run ./tests/software/dsh -entry test_dsh_filterbed -c tests/config/nextflow.config tags: - dsh - dsh_filterbed @@ -8,7 +8,7 @@ md5sum: 91cee4392d6034793f99f6c53891d866 - name: Run dsh splitbed - command: nextflow run ./tests/software/dsh -profile docker -entry test_dsh_splitbed -c tests/config/nextflow.config + command: nextflow run ./tests/software/dsh -entry test_dsh_splitbed -c tests/config/nextflow.config tags: - dsh - dsh_splitbed diff --git a/tests/software/fastp/test.yml b/tests/software/fastp/test.yml index 56cbcaa7..5034c69a 100644 --- a/tests/software/fastp/test.yml +++ b/tests/software/fastp/test.yml @@ -1,5 +1,5 @@ - name: fastp_se - command: nextflow run ./tests/software/fastp -profile docker -entry test_fastp_se -c ./tests/config/nextflow.config + command: nextflow run ./tests/software/fastp -entry test_fastp_se -c ./tests/config/nextflow.config tags: - fastp - fastp_se @@ -12,7 +12,7 @@ - path: ./output/fastp/test.fastp.html - name: fastp_pe - command: nextflow run ./tests/software/fastp -profile docker -entry test_fastp_pe -c ./tests/config/nextflow.config + command: nextflow run ./tests/software/fastp -entry test_fastp_pe -c ./tests/config/nextflow.config tags: - fastp - fastp_pe diff --git a/tests/software/fastqc/test.yml b/tests/software/fastqc/test.yml index f0f6b420..2c0862dd 100644 --- a/tests/software/fastqc/test.yml +++ b/tests/software/fastqc/test.yml @@ -1,5 +1,5 @@ - name: Run fastqc single-end test workflow - command: nextflow run ./tests/software/fastqc/ -profile docker -entry test_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/fastqc/ -entry test_single_end -c tests/config/nextflow.config tags: - fastqc files: @@ -7,7 +7,7 @@ - path: output/test_single_end/test_fastqc.zip - name: Run fastqc paired-end test workflow - command: nextflow run ./tests/software/fastqc/ -profile docker -entry test_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/fastqc/ -entry test_paired_end -c tests/config/nextflow.config tags: - fastqc files: diff --git a/tests/software/gffread/test.yml b/tests/software/gffread/test.yml index daf0cde8..a05e609d 100644 --- a/tests/software/gffread/test.yml +++ b/tests/software/gffread/test.yml @@ -1,5 +1,5 @@ - name: Run gffread test workflow - command: nextflow run ./tests/software/gffread/ -profile docker -entry test_gffread -c tests/config/nextflow.config + command: nextflow run ./tests/software/gffread/ -entry test_gffread -c tests/config/nextflow.config tags: - gffread files: diff --git a/tests/software/multiqc/test.yml b/tests/software/multiqc/test.yml index 7fc6d7a3..21dc36df 100644 --- a/tests/software/multiqc/test.yml +++ b/tests/software/multiqc/test.yml @@ -1,5 +1,5 @@ - name: Run multiqc test workflow - command: nextflow run ./tests/software/multiqc/ -profile docker -entry test_multiqc -c tests/config/nextflow.config + command: nextflow run ./tests/software/multiqc/ -entry test_multiqc -c tests/config/nextflow.config tags: - multiqc files: diff --git a/tests/software/pangolin/test.yml b/tests/software/pangolin/test.yml index 2f44feb4..c50b4245 100644 --- a/tests/software/pangolin/test.yml +++ b/tests/software/pangolin/test.yml @@ -1,5 +1,5 @@ - name: pangolin - command: nextflow run ./tests/software/pangolin -profile docker -entry test_pangolin -c ./tests/config/nextflow.config + command: nextflow run ./tests/software/pangolin -entry test_pangolin -c ./tests/config/nextflow.config tags: - pangolin files: diff --git a/tests/software/picard/test.yml b/tests/software/picard/test.yml index 91f62656..62554607 100644 --- a/tests/software/picard/test.yml +++ b/tests/software/picard/test.yml @@ -1,5 +1,5 @@ - name: Run picard mergesamfiles - command: nextflow run ./tests/software/picard -profile docker -entry test_picard_mergesamfiles -c tests/config/nextflow.config + command: nextflow run ./tests/software/picard -entry test_picard_mergesamfiles -c tests/config/nextflow.config tags: - picard - picard_mergesamfiles @@ -8,7 +8,7 @@ md5sum: a9c8a28d1aac362df5973ba2d9923f86 - name: Run picard collectmultiplemetrics - command: nextflow run ./tests/software/picard -profile docker -entry test_picard_collectmultiplemetrics -c tests/config/nextflow.config + command: nextflow run ./tests/software/picard -entry test_picard_collectmultiplemetrics -c tests/config/nextflow.config tags: - picard - picard_collectmultiplemetrics @@ -25,7 +25,7 @@ - path: output/picard/test.CollectMultipleMetrics.quality_distribution_metrics - name: Run picard MarkDuplicates - command: nextflow run ./tests/software/picard -profile docker -entry test_picard_markduplicates -c tests/config/nextflow.config + command: nextflow run ./tests/software/picard -entry test_picard_markduplicates -c tests/config/nextflow.config tags: - picard - picard_markduplicates @@ -35,7 +35,7 @@ md5sum: 0ed0bfc94069380334ccd595f6d8207c - name: Run picard MarkDuplicates without a sorted bam file so it fails - command: nextflow run ./tests/software/picard -profile docker -entry test_picard_markduplicates_not_sorted -c tests/config/nextflow.config + command: nextflow run ./tests/software/picard -entry test_picard_markduplicates_not_sorted -c tests/config/nextflow.config tags: - picard - picard_markduplicates diff --git a/tests/software/preseq/test.yml b/tests/software/preseq/test.yml index 14d940da..40eb6cdc 100644 --- a/tests/software/preseq/test.yml +++ b/tests/software/preseq/test.yml @@ -1,5 +1,5 @@ - name: Run preseq single-end lcextrap - command: nextflow run ./tests/software/preseq -profile docker -entry test_preseq_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/preseq -entry test_preseq_single_end -c tests/config/nextflow.config tags: - preseq - preseq_lcextrap @@ -10,7 +10,7 @@ - path: output/test_preseq_single_end/test.command.log - name: Run preseq paired-end lcextrap - command: nextflow run ./tests/software/preseq -profile docker -entry test_preseq_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/preseq -entry test_preseq_paired_end -c tests/config/nextflow.config tags: - preseq - preseq_lcextrap diff --git a/tests/software/quast/test.yml b/tests/software/quast/test.yml index 2ceccc54..17a7f056 100644 --- a/tests/software/quast/test.yml +++ b/tests/software/quast/test.yml @@ -1,5 +1,5 @@ - name: quast_ref - command: nextflow run ./tests/software/quast -profile docker -entry test_quast_ref -c ./tests/config/nextflow.config + command: nextflow run ./tests/software/quast -entry test_quast_ref -c ./tests/config/nextflow.config tags: - quast - quast_reference @@ -88,7 +88,7 @@ - path: ./output/quast/quast/aligned_stats/NAx_plot.pdf - name: quast_noref - command: nextflow run ./tests/software/quast -profile docker -entry test_quast_noref -c ./tests/config/nextflow.config + command: nextflow run ./tests/software/quast -entry test_quast_noref -c ./tests/config/nextflow.config tags: - quast - quast_no_reference diff --git a/tests/software/salmon/test.yml b/tests/software/salmon/test.yml index e5d09fe7..9febd8c1 100644 --- a/tests/software/salmon/test.yml +++ b/tests/software/salmon/test.yml @@ -1,5 +1,5 @@ - name: salmon index - command: nextflow run ./tests/software/salmon -profile docker -entry test_salmon_index -c tests/config/nextflow.config + command: nextflow run ./tests/software/salmon -entry test_salmon_index -c tests/config/nextflow.config tags: - salmon - salmon_index @@ -31,7 +31,7 @@ md5sum: 79c4ddf34be3a98d5a7b9d153629a6f7 - name: salmon quant paired end - command: nextflow run ./tests/software/salmon -profile docker -entry test_salmon_quant_pe -c tests/config/nextflow.config + command: nextflow run ./tests/software/salmon -entry test_salmon_quant_pe -c tests/config/nextflow.config tags: - salmon - salmon_quant @@ -85,7 +85,7 @@ md5sum: 27a76542337df436436e66017f66dd25 - name: salmon quant single-end - command: nextflow run ./tests/software/salmon -profile docker -entry test_salmon_quant_se -c tests/config/nextflow.config + command: nextflow run ./tests/software/salmon -entry test_salmon_quant_se -c tests/config/nextflow.config tags: - salmon - salmon_quant diff --git a/tests/software/samtools/test.yml b/tests/software/samtools/test.yml index 5fc3b47c..c198cdd3 100644 --- a/tests/software/samtools/test.yml +++ b/tests/software/samtools/test.yml @@ -1,5 +1,5 @@ - name: Run samtools flagstat test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_flagstat -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_flagstat -c tests/config/nextflow.config tags: - samtools - samtools_flagstat @@ -8,7 +8,7 @@ md5sum: 80590621c74f5ee43ada20d010a3837f - name: Run samtools idxstats test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_idxstats -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_idxstats -c tests/config/nextflow.config tags: - samtools - samtools_idxstats @@ -17,7 +17,7 @@ md5sum: 3bee8b7d55e412da2b3816c5a18d60d5 - name: Run samtools index test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_index -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_index -c tests/config/nextflow.config tags: - samtools - samtools_index @@ -26,7 +26,7 @@ md5sum: 67338c2995eae82c849830ca55aa7bd1 - name: Run samtools sort test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_sort -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_sort -c tests/config/nextflow.config tags: - samtools - samtools_sort @@ -35,7 +35,7 @@ md5sum: a41bfadacd2eeef1d31e05c135cc4f4e - name: Run samtools stats test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_stats -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_stats -c tests/config/nextflow.config tags: - samtools - samtools_stats @@ -44,7 +44,7 @@ md5sum: 06c183864d6e47ab89b0650cae831a93 - name: Run samtools view test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_view -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_view -c tests/config/nextflow.config tags: - samtools - samtools_view @@ -53,7 +53,7 @@ md5sum: 7ea2c325e2249ed7ef44e1d9fdc4f8ff - name: Run samtools mpileup test workflow - command: nextflow run ./tests/software/samtools/ -profile docker -entry test_samtools_mpileup -c tests/config/nextflow.config + command: nextflow run ./tests/software/samtools/ -entry test_samtools_mpileup -c tests/config/nextflow.config tags: - samtools - samtools_mpileup diff --git a/tests/software/seacr/test.yml b/tests/software/seacr/test.yml index 88568d35..ed8306b1 100644 --- a/tests/software/seacr/test.yml +++ b/tests/software/seacr/test.yml @@ -1,5 +1,5 @@ - name: Run seacr call peak test workflow - command: nextflow run ./tests/software/seacr/ -profile docker -entry test_seacr_callpeak -c tests/config/nextflow.config + command: nextflow run ./tests/software/seacr/ -entry test_seacr_callpeak -c tests/config/nextflow.config tags: - seacr - seacr_callpeak diff --git a/tests/software/star/test.yml b/tests/software/star/test.yml index fa7363fb..aca9595b 100644 --- a/tests/software/star/test.yml +++ b/tests/software/star/test.yml @@ -1,5 +1,5 @@ - name: Run star genomegenerate - command: nextflow run ./tests/software/star -profile docker -entry test_star_genomegenerate -c tests/config/nextflow.config + command: nextflow run ./tests/software/star -entry test_star_genomegenerate -c tests/config/nextflow.config tags: - star - star_genomegenerate @@ -32,7 +32,7 @@ md5sum: 8fbe69abbbef4f89da3854873984dbac - name: Run star single-end alignment - command: nextflow run ./tests/software/star -profile docker -entry test_star_alignment_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/star -entry test_star_alignment_single_end -c tests/config/nextflow.config tags: - star - star_align @@ -70,7 +70,7 @@ md5sum: d41d8cd98f00b204e9800998ecf8427e - name: Run star paired-end alignment - command: nextflow run ./tests/software/star -profile docker -entry test_star_alignment_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/star -entry test_star_alignment_paired_end -c tests/config/nextflow.config tags: - star - star_align diff --git a/tests/software/stringtie/test.yml b/tests/software/stringtie/test.yml index 21cea0e8..5bbf002b 100644 --- a/tests/software/stringtie/test.yml +++ b/tests/software/stringtie/test.yml @@ -1,5 +1,5 @@ - name: Run stringtie forward strand test workflow - command: nextflow run ./tests/software/stringtie/ -profile docker -entry test_stringtie_forward -c tests/config/nextflow.config + command: nextflow run ./tests/software/stringtie/ -entry test_stringtie_forward -c tests/config/nextflow.config tags: - stringtie files: @@ -21,7 +21,7 @@ md5sum: 0106f70121a8b520d98b8739aed92915 - name: Run stringtie reverse strand test workflow - command: nextflow run ./tests/software/stringtie/ -profile docker -entry test_stringtie_reverse -c tests/config/nextflow.config + command: nextflow run ./tests/software/stringtie/ -entry test_stringtie_reverse -c tests/config/nextflow.config tags: - stringtie files: diff --git a/tests/software/trimgalore/test.yml b/tests/software/trimgalore/test.yml index 9ba4dddf..34d880db 100644 --- a/tests/software/trimgalore/test.yml +++ b/tests/software/trimgalore/test.yml @@ -1,5 +1,5 @@ - name: Run trimgalore single-end test workflow - command: nextflow run ./tests/software/trimgalore/ -profile docker -entry test_trimgalore_single_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/trimgalore/ -entry test_trimgalore_single_end -c tests/config/nextflow.config tags: - trimgalore files: @@ -9,7 +9,7 @@ - path: output/test_single_end/test_trimmed.fq.gz - name: Run trimgalore paired-end test workflow - command: nextflow run ./tests/software/trimgalore/ -profile docker -entry test_trimgalore_paired_end -c tests/config/nextflow.config + command: nextflow run ./tests/software/trimgalore/ -entry test_trimgalore_paired_end -c tests/config/nextflow.config tags: - trimgalore files: diff --git a/tests/software/ucsc/test.yml b/tests/software/ucsc/test.yml index 53be4e77..fae05063 100644 --- a/tests/software/ucsc/test.yml +++ b/tests/software/ucsc/test.yml @@ -1,5 +1,5 @@ - name: Run bedgraphtobigwig test workflow - command: nextflow run ./tests/software/ucsc/ -profile docker -entry test_ucsc_bedgraphtobigwig -c tests/config/nextflow.config + command: nextflow run ./tests/software/ucsc/ -entry test_ucsc_bedgraphtobigwig -c tests/config/nextflow.config tags: - ucsc_bedgraphtobigwig files: From 654c6225f01de7a6ab916886df4b4bc60ff73320 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:38:51 -0600 Subject: [PATCH 05/20] ci: Setup multiple profiles for testing Added conda and singularity --- .github/workflows/pytest-workflow.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index b3775870..50e8c53c 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -24,6 +24,7 @@ jobs: matrix: nxf_version: ['20.11.0-edge'] tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] + profile: ['docker', 'singularity', 'conda' ] env: NXF_ANSI_LOG: false steps: @@ -59,9 +60,22 @@ jobs: - name: Install dependencies run: python -m pip install --upgrade pip pytest-workflow + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.6.4 + + - name: Setup conda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + # Test the module - name: Run pytest-workflow - run: PROFILE=docker pytest --tag ${{ matrix.tags }} --symlink --wt 2 + run: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --wt 2 - name: Upload logs on failure if: ${{ failure() }} From 1e9aad430d8fac8a64e945657a8bbc6db3a23ed0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:40:30 -0600 Subject: [PATCH 06/20] ci: Remove ${{}} for style points It's not necessary in if statements --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 50e8c53c..9bd2d4c6 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -78,7 +78,7 @@ jobs: run: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --wt 2 - name: Upload logs on failure - if: ${{ failure() }} + if: failure() uses: actions/upload-artifact@v2 with: name: logs From bf2e027fdf035d0fc106f53c2b9368f0ed733c20 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:44:00 -0600 Subject: [PATCH 07/20] ci: Remove conda to kickoff CI --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 9bd2d4c6..d485d88f 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -24,7 +24,7 @@ jobs: matrix: nxf_version: ['20.11.0-edge'] tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] - profile: ['docker', 'singularity', 'conda' ] + profile: ['docker', 'singularity'] #'conda' ] env: NXF_ANSI_LOG: false steps: From 8cd6e41df98b6c6f648e793c6249ec8eb78dca78 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 15:52:43 -0600 Subject: [PATCH 08/20] ci: Change singularity version 3.6.4 => 3.5.3 Per @grst having issues after 3.6 --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index d485d88f..937afe6d 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -64,7 +64,7 @@ jobs: if: matrix.profile == 'singularity' uses: eWaterCycle/setup-singularity@v5 with: - singularity-version: 3.6.4 + singularity-version: 3.5.3 - name: Setup conda if: matrix.profile == 'conda' From 983b02401064af904db73f4509506c2d1e3ca5dc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 16:26:24 -0600 Subject: [PATCH 09/20] ci: Use ubuntu-20.04 for singularity --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 937afe6d..b273de19 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -16,7 +16,7 @@ jobs: filters: '.github/filters.yml' test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 name: CI Test needs: changes if: needs.changes.outputs.modules != '[]' From dcfd1089e5016deac64a55f8c062de91ecd53688 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 16:32:31 -0600 Subject: [PATCH 10/20] ci: Add --keep-workflow-wd-on-fail --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index b273de19..001c204c 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -75,7 +75,7 @@ jobs: # Test the module - name: Run pytest-workflow - run: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --wt 2 + run: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --wt 2 --kwdof - name: Upload logs on failure if: failure() From bb3196ddeb1f65b5a2da0c4037d0f1de26cd253f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 16:46:56 -0600 Subject: [PATCH 11/20] ci: Update name to use matrix --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 001c204c..4eb86980 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -17,7 +17,7 @@ jobs: test: runs-on: ubuntu-20.04 - name: CI Test + name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.nxf_version }} needs: changes if: needs.changes.outputs.modules != '[]' strategy: From c63ba067772573993f06cba9ed92149923298393 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 16:47:15 -0600 Subject: [PATCH 12/20] ci: Only test docker for now I can't access the artifacts. --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 4eb86980..727cf0d5 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -24,7 +24,7 @@ jobs: matrix: nxf_version: ['20.11.0-edge'] tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] - profile: ['docker', 'singularity'] #'conda' ] + profile: ['docker'] # FIXME, 'singularity', 'conda' ] env: NXF_ANSI_LOG: false steps: From 6b675eef6d2dbc1a5fe47dc9a27893f0a6b7a2c0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:00:40 -0600 Subject: [PATCH 13/20] ci: Set pull docker container to false Co-authored-by: Harshil Patel --- tests/config/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 9af14ccf..babc8735 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -2,6 +2,7 @@ params { outdir = "output/" publish_dir_mode = "copy" enable_conda = false + singularity_pull_docker_container = false } if("$PROFILE" == "docker") { From 725de25ce414a76ec660c5f1eebfe044b82a82af Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:03:29 -0600 Subject: [PATCH 14/20] Revert "ci: Only test docker for now" This reverts commit c63ba067772573993f06cba9ed92149923298393. --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 727cf0d5..4eb86980 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -24,7 +24,7 @@ jobs: matrix: nxf_version: ['20.11.0-edge'] tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] - profile: ['docker'] # FIXME, 'singularity', 'conda' ] + profile: ['docker', 'singularity'] #'conda' ] env: NXF_ANSI_LOG: false steps: From 0813e61c01184883993a2ba7d1403d6081482618 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:08:10 -0600 Subject: [PATCH 15/20] ci: Kick off all container types --- .github/workflows/pytest-workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 4eb86980..965fc8fe 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -21,10 +21,11 @@ jobs: needs: changes if: needs.changes.outputs.modules != '[]' strategy: + fail-fast: false matrix: nxf_version: ['20.11.0-edge'] tags: ['${{ fromJson(needs.changes.outputs.modules) }}'] - profile: ['docker', 'singularity'] #'conda' ] + profile: ['docker', 'singularity', 'conda'] env: NXF_ANSI_LOG: false steps: From 42b8962320078a2dbd1520b2caeffee26caf5c7b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:13:35 -0600 Subject: [PATCH 16/20] ci: Add bioconda channels --- .github/workflows/pytest-workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 965fc8fe..08d70aa2 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -67,11 +67,12 @@ jobs: with: singularity-version: 3.5.3 - - name: Setup conda + - name: Setup miniconda if: matrix.profile == 'conda' uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true + channels: defaults, bioconda, conda-forge python-version: ${{ matrix.python-version }} # Test the module From 67deb0d47ad4b2e5afe477479388a1ef3bf40060 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:25:58 -0600 Subject: [PATCH 17/20] ci: Add singularity tmp dir Co-authored-by: Harshil Patel --- tests/config/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index babc8735..6093aa88 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -11,6 +11,7 @@ if("$PROFILE" == "docker") { } else if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true + singularity.runOptions = '-B /tmp/' } else if ("$PROFILE" == "conda") { params.enable_conda = true } From 72b4fb35aa176ce713e6b8a0b381d8ebd90b4336 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:32:21 -0600 Subject: [PATCH 18/20] test: Fix profile logic order --- tests/config/nextflow.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 6093aa88..e8a17bbb 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -5,13 +5,13 @@ params { singularity_pull_docker_container = false } -if("$PROFILE" == "docker") { - docker.enabled = true - docker.runOptions = '-u \$(id -u):\$(id -g)' -} else if ("$PROFILE" == "singularity") { +if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true singularity.runOptions = '-B /tmp/' } else if ("$PROFILE" == "conda") { params.enable_conda = true +} else { + docker.enabled = true + docker.runOptions = '-u \$(id -u):\$(id -g)' } From 54bb0ae7a1f557b83948559ba0ba5a879c5605b9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 17:51:20 -0600 Subject: [PATCH 19/20] ci: Name logs --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 08d70aa2..87f02f55 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -83,7 +83,7 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: logs + name: logs-${{ matrix.tags }}-${{ matrix.profile }}-${{ matrix.nxf_version }} path: | /tmp/pytest_workflow_*/*/.nextflow.log /tmp/pytest_workflow_*/*/log.out From 467d1595c40c357d9114f823c9ba819d487445f3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 2 Feb 2021 18:14:39 -0600 Subject: [PATCH 20/20] test: Use docker image with singularity --- tests/config/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index e8a17bbb..38396a86 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -2,7 +2,7 @@ params { outdir = "output/" publish_dir_mode = "copy" enable_conda = false - singularity_pull_docker_container = false + singularity_pull_docker_container = true } if ("$PROFILE" == "singularity") {