Add bcl2fastq2 to cellranger dockerfile (#1115)

* refactor: add bcl2fastq to cellranger dockerfile

bcl2fastq required for the cellranger mkfastq module and was therefore
added to the cellranger dockerfile. Further, cellranger was updated to
the latest version, 6.1.2., with naming and containers across
cellranger modules updated accordingly.

* chore: add bcl2fastq zip to .gitignore

* style: fix code linting error

* test(cellranger): Add tiles to mkfastq

* additional dockerfile for mkfastq

* update readme and dockerfiles

* update readme

* fix: update container for mkfastq

* docs: correct typos in readme

* test: update md5sum following cellranger update

* test: update md5sum following cellranger update

* fix: new line for external args in mkfastq

* test: update mkfastq tiles argument

* test: comment out mkfastq tests until smaller test data found

* test: stub-run mkfastq test until smaller test data found

* test: fix incorrect file path for mkfastq

Co-authored-by: Edmund Miller <edmund.a.miller@protonmail.com>
Co-authored-by: ggabernet <gisela.gabernet@qbic.uni-tuebingen.de>
Co-authored-by: Gisela Gabernet <gisela.gabernet@gmail.com>

modules/cellranger/.gitignore

@@ -1 +1,2 @@
 cellranger-*.tar.gz
+bcl2fastq2-*.zip

modules/cellranger/Dockerfile

@@ -1,15 +1,22 @@
+# Dockerfile to create container with Cell Ranger v6.1.2
+# Push to nfcore/cellranger:<VER>
+
 FROM continuumio/miniconda3:4.8.2
 LABEL authors="Gisela Gabernet <gisela.gabernet@gmail.com>" \
     description="Docker image containing Cell Ranger"
-# Disclaimer: this container is not provided nor supported by 10x Genomics.
+# Disclaimer: this container is not provided nor supported by Illumina or 10x Genomics.
 
 # Install procps and clean apt cache
 RUN apt-get update --allow-releaseinfo-change \
-    && apt-get install -y procps \
+    && apt-get install -y \
+    cpio \
+    procps \
+    rpm2cpio \
+    unzip \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # Copy pre-downloaded cellranger file
-ENV CELLRANGER_VER 6.0.2
+ENV CELLRANGER_VER=6.1.2
 COPY cellranger-$CELLRANGER_VER.tar.gz /opt/cellranger-$CELLRANGER_VER.tar.gz
 
 # Install cellranger

modules/cellranger/README.md

@@ -1,18 +1,24 @@
 # Updating the docker container and making a new module release
 
-Cell Ranger is a commercial tool by 10X Genomics. The container provided for the cellranger nf-core module is not provided nor supported by 10x Genomics. Updating the Cell Ranger version in the container and pushing the update to Dockerhub needs to be done manually.
+Cell Ranger is a commercial tool from 10X Genomics. The container provided for the cellranger nf-core module is not provided nor supported by 10x Genomics. Updating the Cell Ranger versions in the container and pushing the update to Dockerhub needs to be done manually.
 
-1. Navigate to the [Cell Ranger download page](https://support.10xgenomics.com/single-cell-gene-expression/software/downloads/latest) and download the tar ball of the desired Cell Ranger version with `curl` or `wget`. Place this file in the same folder where the Dockerfile lies.
+1. Navigate to the appropriate download page.
+        - [Cell Ranger](https://support.10xgenomics.com/single-cell-gene-expression/software/downloads/latest): download the tar ball of the desired Cell Ranger version with `curl` or `wget`. Place this file in the same folder where the Dockerfile lies.
 
-2. Edit the Dockerfile: update the Cell Ranger version in this line:
+2. Edit the Dockerfile. Update the Cell Ranger versions in this line:
 
     ```bash
-    ENV CELLRANGER_VER <VERSION>
+    ENV CELLRANGER_VER=<VERSION>
     ```
 
-3. Create the container:
+3. Create and test the container:
 
     ```bash
     docker build . -t nfcore/cellranger:<VERSION>
+    ```
+
+4. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so.
+
+    ```bash
     docker push nfcore/cellranger:<VERSION>
     ```

modules/cellranger/count/main.nf

@@ -5,7 +5,7 @@ process CELLRANGER_COUNT {
     if (params.enable_conda) {
         exit 1, "Conda environments cannot be used when using the Cell Ranger tool. Please use docker or singularity containers."
     }
-    container "nfcore/cellranger:6.0.2"
+    container "nfcore/cellranger:6.1.2"
 
     input:
     tuple val(meta), path(reads)

modules/cellranger/mkfastq/Dockerfile

@@ -0,0 +1,40 @@
+# Dockerfile to create container with Cell Ranger v6.1.2 and bcl2fastq v2.20.0
+# Push to nfcore/cellrangermkfastq:<VER>
+
+FROM continuumio/miniconda3:4.8.2
+LABEL authors="Regina Reynolds, Gisela Gabernet <gisela.gabernet@gmail.com>" \
+    description="Docker image containing bcl2fastq2 and Cell Ranger"
+# Disclaimer: this container is not provided nor supported by Illumina or 10x Genomics.
+
+# Install procps and clean apt cache
+RUN apt-get update --allow-releaseinfo-change \
+    && apt-get install -y \
+    cpio \
+    procps \
+    rpm2cpio \
+    unzip \
+    && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+
+# Copy pre-downloaded bcl2fastq2 and cellranger file
+ENV BCL2FASTQ2_VER=v2-20-0-linux-x86-64 \
+    CELLRANGER_VER=6.1.2
+COPY bcl2fastq2-$BCL2FASTQ2_VER.zip /tmp/bcl2fastq2-$BCL2FASTQ2_VER.zip
+COPY cellranger-$CELLRANGER_VER.tar.gz /opt/cellranger-$CELLRANGER_VER.tar.gz
+
+# Install bcl2fastq2
+RUN \
+    cd /tmp && \
+    unzip bcl2fastq2-$BCL2FASTQ2_VER.zip && \
+    mv *.rpm bcl2fastq2-$BCL2FASTQ2_VER.rpm && \
+    rpm2cpio ./bcl2fastq2-$BCL2FASTQ2_VER.rpm | cpio -idmv && \
+    export PATH=/tmp/usr/local/bin/:$PATH && \
+    ln -s /tmp/usr/local/bin/bcl2fastq /usr/bin/bcl2fastq && \
+    rm -rf bcl2fastq2-$BCL2FASTQ2_VER.*
+
+# Install cellranger
+RUN \
+    cd /opt && \
+    tar -xzvf cellranger-$CELLRANGER_VER.tar.gz && \
+    export PATH=/opt/cellranger-$CELLRANGER_VER:$PATH && \
+    ln -s /opt/cellranger-$CELLRANGER_VER/cellranger /usr/bin/cellranger && \
+    rm -rf /opt/cellranger-$CELLRANGER_VER.tar.gz

modules/cellranger/mkfastq/README.md

@@ -0,0 +1,26 @@
+# Updating the docker container and making a new module release
+
+Bcl2fastq2 and Cell Ranger are commercial tools from Illumina and 10X Genomics, respectively. The container provided for the cellranger nf-core module is not provided nor supported by either Illumina or 10x Genomics. Updating the bcl2fastq2 or Cell Ranger versions in the container and pushing the update to Dockerhub needs to be done manually.
+
+1. Navigate to the appropriate download pages.
+        - [bcl2fastq2](https://emea.support.illumina.com/sequencing/sequencing_software/bcl2fastq-conversion-software.html): download the linux rpm installer of the desired bcl2fastq2 version with `curl` or `wget`. Place this file in the same folder where the Dockerfile lies.
+        - [Cell Ranger](https://support.10xgenomics.com/single-cell-gene-expression/software/downloads/latest): download the tar ball of the desired Cell Ranger version with `curl` or `wget`. Place this file in the same folder where the Dockerfile lies.
+
+2. Edit the Dockerfile. Update the bcl2fastq2 and Cell Ranger versions in this line:
+
+    ```bash
+    ENV BCL2FASTQ2_VER=<VERSION> \
+        CELLRANGER_VER=<VERSION>
+    ```
+
+3. Create and test the container:
+
+    ```bash
+    docker build . -t nfcore/cellrangermkfastq:<CELLRANGER_VERSION>
+    ```
+
+4. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so.
+
+    ```bash
+    docker push nfcore/cellrangermkfastq:<CELLRANGER_VERSION>
+    ```

modules/cellranger/mkfastq/main.nf

@@ -5,7 +5,7 @@ process CELLRANGER_MKFASTQ {
     if (params.enable_conda) {
         exit 1, "Conda environments cannot be used when using the Cell Ranger tool. Please use docker or singularity containers."
     }
-    container "litd/docker-cellranger:v6.1.1" // FIXME Add bcl2fastq to nf-core docker image
+    container "nfcore/cellrangermkfastq:6.1.2"
 
     input:
     path bcl
@@ -13,14 +13,14 @@ process CELLRANGER_MKFASTQ {
 
     output:
     path "versions.yml", emit: versions
-    path "*.fastq.gz"  , emit: fastq
+    path "${bcl.getSimpleName()}/outs/fastq_path/*.fastq.gz"  , emit: fastq
 
     script:
     def args = task.ext.args ?: ''
     """
     cellranger mkfastq --id=${bcl.getSimpleName()} \
         --run=$bcl \
-        --csv=$csv
+        --csv=$csv \
         $args
 
     cat <<-END_VERSIONS > versions.yml
@@ -28,4 +28,15 @@ process CELLRANGER_MKFASTQ {
         cellranger: \$(echo \$( cellranger --version 2>&1) | sed 's/^.*[^0-9]\\([0-9]*\\.[0-9]*\\.[0-9]*\\).*\$/\\1/' )
     END_VERSIONS
     """
+
+    stub:
+    """
+    mkdir -p "${bcl.getSimpleName()}/outs/fastq_path/"
+    touch ${bcl.getSimpleName()}/outs/fastq_path/fake_file.fastq.gz
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        cellranger: \$(echo \$( cellranger --version 2>&1) | sed 's/^.*[^0-9]\\([0-9]*\\.[0-9]*\\.[0-9]*\\).*\$/\\1/' )
+    END_VERSIONS
+    """
 }

modules/cellranger/mkgtf/main.nf

@@ -5,7 +5,7 @@ process CELLRANGER_MKGTF {
     if (params.enable_conda) {
         exit 1, "Conda environments cannot be used when using the Cell Ranger tool. Please use docker or singularity containers."
     }
-    container "nfcore/cellranger:6.0.2"
+    container "nfcore/cellranger:6.1.2"
 
     input:
     path gtf

modules/cellranger/mkref/main.nf

@@ -5,7 +5,7 @@ process CELLRANGER_MKREF {
     if (params.enable_conda) {
         exit 1, "Conda environments cannot be used when using the Cell Ranger tool. Please use docker or singularity containers."
     }
-    container "nfcore/cellranger:6.0.2"
+    container "nfcore/cellranger:6.1.2"
 
     input:
     path fasta

tests/modules/cellranger/count/test.yml

@@ -8,12 +8,12 @@
     - path: output/cellranger/sample-123/outs/metrics_summary.csv
       md5sum: 707df0f101d479d93f412ca74f9c4131
     - path: output/cellranger/sample-123/outs/molecule_info.h5
-      md5sum: cf03b2b3ca776a1c37aa3518e91268ba
+      md5sum: 0e56836ef0725f2ab05f56ca5a71e55b
     - path: output/cellranger/sample-123/outs/possorted_genome_bam.bam
       md5sum: 15441da9cfceea0bb48c8b66b1b860df
     - path: output/cellranger/sample-123/outs/possorted_genome_bam.bam.bai
       md5sum: 7c3d49c77016a09535aff61a027f750c
     - path: output/cellranger/sample-123/outs/raw_feature_bc_matrix
     - path: output/cellranger/sample-123/outs/raw_feature_bc_matrix.h5
-      md5sum: 40c8df814eb8723b7317b234dc8222e9
+      md5sum: cdad1cd7b215d7137cf92515e81a8525
     - path: output/cellranger/sample-123/outs/web_summary.html

tests/modules/cellranger/mkfastq/nextflow.config

@@ -2,4 +2,8 @@ process {
 
     publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
 
+    withName: CELLRANGER_MKFASTQ {
+        ext.args = "--tiles 1101"
+    }
+
 }

tests/modules/cellranger/mkfastq/test.yml

@@ -1,13 +1,16 @@
 - name: cellranger mkfastq test_cellranger_mkfastq_simple
-  command: nextflow run tests/modules/cellranger/mkfastq -entry test_cellranger_mkfastq_simple -c tests/config/nextflow.config -c ./tests/modules/cellranger/mkfastq/nextflow.config
+  command: nextflow run tests/modules/cellranger/mkfastq -entry test_cellranger_mkfastq_simple -c tests/config/nextflow.config -c ./tests/modules/cellranger/mkfastq/nextflow.config -stub-run
   tags:
     - cellranger
     - cellranger/mkfastq
-  # files:
+  files:
-  #   - path: output/cellranger/genome.filtered.gtf
+    - path: output/cellranger/cellranger-tiny-bcl-1/outs/fastq_path/fake_file.fastq.gz
-  #     md5sum: a8b8a7b5039e05d3a9cf9151ea138b5b
+      md5sum: d41d8cd98f00b204e9800998ecf8427e
 - name: cellranger mkfastq test_cellranger_mkfastq_illumina
-  command: nextflow run tests/modules/cellranger/mkfastq -entry test_cellranger_mkfastq_illumina -c tests/config/nextflow.config -c ./tests/modules/cellranger/mkfastq/nextflow.config
+  command: nextflow run tests/modules/cellranger/mkfastq -entry test_cellranger_mkfastq_illumina -c tests/config/nextflow.config -c ./tests/modules/cellranger/mkfastq/nextflow.config -stub-run
   tags:
     - cellranger
     - cellranger/mkfastq
+  files:
+    - path: output/cellranger/cellranger-tiny-bcl-1/outs/fastq_path/fake_file.fastq.gz
+      md5sum: d41d8cd98f00b204e9800998ecf8427e

tests/modules/cellranger/mkref/test.yml

@@ -11,7 +11,7 @@
     - path: output/cellranger/homo_sapiens_chr22_reference/genes/genes.gtf.gz
       md5sum: 6d9b5f409bfea95022bc25b9590e194e
     - path: output/cellranger/homo_sapiens_chr22_reference/reference.json
-      md5sum: a4e2b9bbf016c55b0d4d7bc1fa53896f
+      md5sum: 5d8d1669cd251433505f183e1c9ed6bc
     - path: output/cellranger/homo_sapiens_chr22_reference/star/Genome
       md5sum: 22102926fadf5890e905ca71b2da3f35
     - path: output/cellranger/homo_sapiens_chr22_reference/star/SA