Notebook modules (#617)

* Draft rmarkdown module

* stub jupyter notebook module

* Create yaml file with params

* Update meta.yml for rmarkdown module

* Add comment on YAML

* Update notebooks module, clean up parametrize.nf

* Two separate channels for parameters and input files

* Fix Rmd render script

* Add tests for rmarkdown

* Fix tests for rmarkdown module

* Update checksums

* Fix tests for jupyter

* Test without Grab()

* Update software versions

* update rmarkdown dependencies

* Draft for multiple versions

* Fix indent of script

* Fix indent in rmarkdown script

* Emit version.syml

* Update modules/rmarkdown/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/rmarkdown/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/rmarkdown/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Rename rmarkdown to rmarkdownnotebook

* Add rmarkdown mulled biocontainer

* Write sessionInfo to separate log file

* Update rmarkdownnotebook

* Sessioninfo does not have a stable md5sum

* Update jupyternotebook

* Update meta

* Add jupyternotebook biocontainers

* Handle Groovy Gstrings in parameterize

* Update to versions.yml

* Update functions.nf

* Fix versions yaml

* Fix EC lint

* Update modules/rmarkdownnotebook/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/jupyternotebook/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Use official test data

* Harshilify

* Make parameters channel clearer

* Apply suggestions from code review

Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>

* Apply suggestions from code review

* Update main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>
Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>

modules/jupyternotebook/functions.nf

@@ -0,0 +1,78 @@
+//
+//  Utility functions used in nf-core DSL2 module files
+//
+
+//
+// Extract name of software tool from process name using $task.process
+//
+def getSoftwareName(task_process) {
+    return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
+}
+
+//
+// Extract name of module from process name using $task.process
+//
+def getProcessName(task_process) {
+    return task_process.tokenize(':')[-1]
+}
+
+//
+// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
+//
+def initOptions(Map args) {
+    def Map options = [:]
+    options.args            = args.args ?: ''
+    options.args2           = args.args2 ?: ''
+    options.args3           = args.args3 ?: ''
+    options.publish_by_meta = args.publish_by_meta ?: []
+    options.publish_dir     = args.publish_dir ?: ''
+    options.publish_files   = args.publish_files
+    options.suffix          = args.suffix ?: ''
+    return options
+}
+
+//
+// Tidy up and join elements of a list to return a path string
+//
+def getPathFromList(path_list) {
+    def paths = path_list.findAll { item -> !item?.trim().isEmpty() }      // Remove empty entries
+    paths     = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
+    return paths.join('/')
+}
+
+//
+// Function to save/publish module results
+//
+def saveFiles(Map args) {
+    def ioptions  = initOptions(args.options)
+    def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
+
+    // Do not publish versions.yml unless running from pytest workflow
+    if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
+        return null
+    }
+    if (ioptions.publish_by_meta) {
+        def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
+        for (key in key_list) {
+            if (args.meta && key instanceof String) {
+                def path = key
+                if (args.meta.containsKey(key)) {
+                    path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
+                }
+                path = path instanceof String ? path : ''
+                path_list.add(path)
+            }
+        }
+    }
+    if (ioptions.publish_files instanceof Map) {
+        for (ext in ioptions.publish_files) {
+            if (args.filename.endsWith(ext.key)) {
+                def ext_list = path_list.collect()
+                ext_list.add(ext.value)
+                return "${getPathFromList(ext_list)}/$args.filename"
+            }
+        }
+    } else if (ioptions.publish_files == null) {
+        return "${getPathFromList(path_list)}/$args.filename"
+    }
+}

modules/jupyternotebook/main.nf

@@ -0,0 +1,92 @@
+// Import generic module functions
+include { initOptions; saveFiles; getProcessName; getSoftwareName } from './functions'
+include { dump_params_yml; indent_code_block } from "./parametrize"
+
+params.options         = [:]
+options                = initOptions(params.options)
+params.parametrize     = true
+params.implicit_params = true
+params.meta_params     = true
+
+process JUPYTERNOTEBOOK {
+    tag "$meta.id"
+    label 'process_low'
+    publishDir "${params.outdir}",
+        mode: params.publish_dir_mode,
+        saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
+
+    //NB: You likely want to override this with a container containing all required
+    //dependencies for your analysis. The container at least needs to contain the
+    //ipykernel, jupytext, papermill and nbconvert Python packages.
+    conda (params.enable_conda ? "ipykernel=6.0.3 jupytext=1.11.4 nbconvert=6.1.0 papermill=2.3.3 matplotlib=3.4.2" : null)
+    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
+        container "https://depot.galaxyproject.org/singularity/mulled-v2-514b1a5d280c7043110b2a8d0a87b57ba392a963%3A879972fc8bdc81ee92f2bce3b4805d89a772bf84-0"
+    } else {
+        container "quay.io/biocontainers/mulled-v2-514b1a5d280c7043110b2a8d0a87b57ba392a963:879972fc8bdc81ee92f2bce3b4805d89a772bf84-0"
+    }
+
+    input:
+    tuple val(meta), path(notebook)
+    val parameters
+    path input_files
+
+    output:
+    tuple val(meta), path("*.html"), emit: report
+    tuple val(meta), path("artifacts/"), emit: artifacts, optional: true
+    path "versions.yml"            , emit: versions
+
+    script:
+    def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
+
+    // Dump parameters to yaml file.
+    // Using a yaml file over using the CLI params because
+    //  * no issue with escaping
+    //  * allows to pass nested maps instead of just single values
+    def params_cmd = ""
+    def render_cmd = ""
+    if (params.parametrize) {
+        nb_params = [:]
+        if (params.implicit_params) {
+            nb_params["cpus"] = task.cpus
+            nb_params["artifact_dir"] = "artifacts"
+            nb_params["input_dir"] = "./"
+        }
+        if (params.meta_params) {
+            nb_params["meta"] = meta
+        }
+        nb_params += parameters
+        params_cmd = dump_params_yml(nb_params)
+        render_cmd = "papermill -f .params.yml"
+    } else {
+        render_cmd = "papermill"
+    }
+
+    """
+    set -o pipefail
+
+    # Dump .params.yml heredoc (section will be empty if parametrization is disabled)
+    ${indent_code_block(params_cmd, 4)}
+
+    # Create output directory
+    mkdir artifacts
+
+    # Set parallelism for BLAS/MKL etc. to avoid over-booking of resources
+    export MKL_NUM_THREADS="${task.cpus}"
+    export OPENBLAS_NUM_THREADS="${task.cpus}"
+    export OMP_NUM_THREADS="${task.cpus}"
+    export NUMBA_NUM_THREADS="${task.cpus}"
+
+    # Convert notebook to ipynb using jupytext, execute using papermill, convert using nbconvert
+    jupytext --to notebook --output - --set-kernel - ${notebook}  \\
+        | ${render_cmd} \\
+        | jupyter nbconvert --stdin --to html --output ${prefix}.html
+
+    cat <<-END_VERSIONS > versions.yml
+    ${getProcessName(task.process)}:
+        jupytext: \$(jupytext --version)
+        ipykernel: \$(python -c "import ipykernel; print(ipykernel.__version__)")
+        nbconvert: \$(jupyter nbconvert --version)
+        papermill: \$(papermill --version | cut -f1 -d' ')
+    END_VERSIONS
+    """
+}

modules/jupyternotebook/meta.yml

@@ -0,0 +1,68 @@
+name: jupyternotebook
+description: |
+  Render jupyter (or jupytext) notebooks to HTML reports. Supports parametrization
+  through papermill.
+keywords:
+  - Python
+  - Jupyter
+  - jupytext
+  - papermill
+  - notebook
+  - reports
+tools:
+  - jupytext:
+      description: Jupyter notebooks as plain text scripts or markdown documents
+      homepage: https://github.com/mwouts/jupytext/
+      documentation: https://jupyter.org/documentation
+      tool_dev_url: https://github.com/mwouts/jupytext/
+      licence: "MIT"
+  - papermill:
+      description: Parameterize, execute, and analyze notebooks
+      homepage: https://github.com/nteract/papermill
+      documentation: http://papermill.readthedocs.io/en/latest/
+      tool_dev_url: https://github.com/nteract/papermill
+      licence: "BSD 3-clause"
+  - nbconvert:
+      description: Parameterize, execute, and analyze notebooks
+      homepage: https://nbconvert.readthedocs.io/en/latest/
+      documentation: https://nbconvert.readthedocs.io/en/latest/
+      tool_dev_url: https://github.com/jupyter/nbconvert
+      licence: "BSD 3-clause"
+
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - notebook:
+      type: file
+      description: Jupyter notebook or jupytext representation thereof
+      pattern: "*.{ipynb,py,md,Rmd,myst}"
+  - parameters:
+      type: map
+      description: |
+        Groovy map with notebook parameters which will be passed
+        to papermill in order to create parametrized reports.
+  - input_files:
+      type: path
+      description: One or multiple files serving as input data for the notebook.
+      pattern: "*"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - report:
+      type: file
+      description: HTML report generated from Jupyter notebook
+      pattern: "*.html"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@grst"

modules/jupyternotebook/parametrize.nf

@@ -0,0 +1,44 @@
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.representer.Representer
+import org.yaml.snakeyaml.DumperOptions
+
+
+/**
+ * Multiline code blocks need to have the same indentation level
+ * as the `script:` section. This function re-indents code to the specified level.
+ */
+def indent_code_block(code, n_spaces) {
+    def indent_str = " ".multiply(n_spaces)
+    return code.stripIndent().split("\n").join("\n" + indent_str)
+}
+
+/**
+ * Create a config YAML file from a groovy map
+ *
+ * @params task The process' `task` variable
+ * @returns a line to be inserted in the bash script.
+ */
+def dump_params_yml(params) {
+    DumperOptions options = new DumperOptions();
+    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+
+    // Properly handle Groovy GStrings
+    // see https://stackoverflow.com/a/35108062/2340703
+    def representer = new Representer() {{
+        this.multiRepresenters.put(GString, this.representers.get(String))
+    }}
+
+    def yaml = new Yaml(representer, options)
+    def yaml_str = yaml.dump(params)
+
+    // Writing the .params.yml file directly as follows does not work.
+    // It only works in 'exec:', but not if there is a `script:` section:
+    // task.workDir.resolve('.params.yml').text = yaml_str
+
+    // Therefore, we inject it into the bash script:
+    return """\
+        cat <<"END_PARAMS_SECTION" > ./.params.yml
+        ${indent_code_block(yaml_str, 8)}
+        END_PARAMS_SECTION
+    """
+}

modules/rmarkdownnotebook/functions.nf

@@ -0,0 +1,78 @@
+//
+//  Utility functions used in nf-core DSL2 module files
+//
+
+//
+// Extract name of software tool from process name using $task.process
+//
+def getSoftwareName(task_process) {
+    return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
+}
+
+//
+// Extract name of module from process name using $task.process
+//
+def getProcessName(task_process) {
+    return task_process.tokenize(':')[-1]
+}
+
+//
+// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
+//
+def initOptions(Map args) {
+    def Map options = [:]
+    options.args            = args.args ?: ''
+    options.args2           = args.args2 ?: ''
+    options.args3           = args.args3 ?: ''
+    options.publish_by_meta = args.publish_by_meta ?: []
+    options.publish_dir     = args.publish_dir ?: ''
+    options.publish_files   = args.publish_files
+    options.suffix          = args.suffix ?: ''
+    return options
+}
+
+//
+// Tidy up and join elements of a list to return a path string
+//
+def getPathFromList(path_list) {
+    def paths = path_list.findAll { item -> !item?.trim().isEmpty() }      // Remove empty entries
+    paths     = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
+    return paths.join('/')
+}
+
+//
+// Function to save/publish module results
+//
+def saveFiles(Map args) {
+    def ioptions  = initOptions(args.options)
+    def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
+
+    // Do not publish versions.yml unless running from pytest workflow
+    if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
+        return null
+    }
+    if (ioptions.publish_by_meta) {
+        def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
+        for (key in key_list) {
+            if (args.meta && key instanceof String) {
+                def path = key
+                if (args.meta.containsKey(key)) {
+                    path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
+                }
+                path = path instanceof String ? path : ''
+                path_list.add(path)
+            }
+        }
+    }
+    if (ioptions.publish_files instanceof Map) {
+        for (ext in ioptions.publish_files) {
+            if (args.filename.endsWith(ext.key)) {
+                def ext_list = path_list.collect()
+                ext_list.add(ext.value)
+                return "${getPathFromList(ext_list)}/$args.filename"
+            }
+        }
+    } else if (ioptions.publish_files == null) {
+        return "${getPathFromList(path_list)}/$args.filename"
+    }
+}

modules/rmarkdownnotebook/main.nf

@@ -0,0 +1,97 @@
+// Import generic module functions
+include { initOptions; saveFiles; getProcessName; getSoftwareName } from './functions'
+include { dump_params_yml; indent_code_block }                      from "./parametrize"
+
+params.options         = [:]
+options                = initOptions(params.options)
+params.parametrize     = true
+params.implicit_params = true
+params.meta_params     = true
+
+process RMARKDOWNNOTEBOOK {
+    tag "$meta.id"
+    label 'process_low'
+    publishDir "${params.outdir}",
+        mode: params.publish_dir_mode,
+        saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
+
+    //NB: You likely want to override this with a container containing all required
+    //dependencies for your analysis. The container at least needs to contain the
+    //yaml and rmarkdown R packages.
+    conda (params.enable_conda ? "r-base=4.1.0 r-rmarkdown=2.9 r-yaml=2.2.1" : null)
+    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
+        container "https://depot.galaxyproject.org/singularity/mulled-v2-31ad840d814d356e5f98030a4ee308a16db64ec5%3A0e852a1e4063fdcbe3f254ac2c7469747a60e361-0"
+    } else {
+        container "quay.io/biocontainers/mulled-v2-31ad840d814d356e5f98030a4ee308a16db64ec5:0e852a1e4063fdcbe3f254ac2c7469747a60e361-0"
+    }
+
+    input:
+    tuple val(meta), path(notebook)
+    val parameters
+    path input_files
+
+    output:
+    tuple val(meta), path("*.html")           , emit: report
+    tuple val(meta), path ("artifacts/*")     , emit: artifacts, optional: true
+    tuple val(meta), path ("session_info.log"), emit: session_info
+    path  "versions.yml"                      , emit: versions
+
+    script:
+    def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
+
+    // Dump parameters to yaml file.
+    // Using a yaml file over using the CLI params because
+    //  * no issue with escaping
+    //  * allows to pass nested maps instead of just single values
+    def params_cmd = ""
+    def render_cmd = ""
+    if (params.parametrize) {
+        nb_params = [:]
+        if (params.implicit_params) {
+            nb_params["cpus"] = task.cpus
+            nb_params["artifact_dir"] = "artifacts"
+            nb_params["input_dir"] = "./"
+        }
+        if (params.meta_params) {
+            nb_params["meta"] = meta
+        }
+        nb_params += parameters
+        params_cmd = dump_params_yml(nb_params)
+        render_cmd = """\
+            params = yaml::read_yaml('.params.yml')
+            rmarkdown::render('${prefix}.Rmd', params=params, envir=new.env())
+        """
+    } else {
+        render_cmd = "rmarkdown::render('${prefix}.Rmd')"
+    }
+
+    """
+    # Dump .params.yml heredoc (section will be empty if parametrization is disabled)
+    ${indent_code_block(params_cmd, 4)}
+
+    # Create output directory
+    mkdir artifacts
+
+    # Set parallelism for BLAS/MKL etc. to avoid over-booking of resources
+    export MKL_NUM_THREADS="${task.cpus}"
+    export OPENBLAS_NUM_THREADS="${task.cpus}"
+    export OMP_NUM_THREADS="${task.cpus}"
+
+    # Work around  https://github.com/rstudio/rmarkdown/issues/1508
+    # If the symbolic link is not replaced by a physical file
+    # output- and temporary files will be written to the original directory.
+    mv "${notebook}" "${notebook}.orig"
+    cp -L "${notebook}.orig" "${prefix}.Rmd"
+
+    # Render notebook
+    Rscript - <<EOF
+        ${indent_code_block(render_cmd, 8)}
+        writeLines(capture.output(sessionInfo()), "session_info.log")
+    EOF
+
+    cat <<-END_VERSIONS > versions.yml
+    ${getProcessName(task.process)}:
+        rmarkdown: \$(Rscript -e "cat(paste(packageVersion('rmarkdown'), collapse='.'))")
+    END_VERSIONS
+    """
+}

modules/rmarkdownnotebook/meta.yml

@@ -0,0 +1,73 @@
+name: rmarkdownnotebook
+description: Render an rmarkdown notebook. Supports parametrization.
+keywords:
+  - R
+  - notebook
+  - reports
+tools:
+  - rmarkdown:
+      description: Dynamic Documents for R
+      homepage: https://rmarkdown.rstudio.com/
+      documentation: https://rmarkdown.rstudio.com/lesson-1.html
+      tool_dev_url: https://github.com/rstudio/rmarkdown
+      doi: ""
+      licence: GPL-3
+
+params:
+  - parametrize:
+      type: boolean
+      description: If true, parametrize the notebook
+  - implicit_params:
+      type: boolean
+      description: |
+        If true (default), include the implicit params
+          * `input_dir`, which points to the directory containing the files added via `input_files`,
+          * `artifact_dir`, which points to the directory where the notebook should place output files, and
+          * `cpus`, which contains the value of ${task.cpus}
+  - meta_params:
+      type: boolean
+      description: |
+        If true, include a parameter `meta` which contains the information specified
+        via the `meta` input channel.
+
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - notebook:
+      type: file
+      description: Rmarkdown file
+      pattern: "*.{Rmd}"
+  - parameters:
+      type: map
+      description: |
+        Groovy map with notebook parameters which will be passed to
+        rmarkdown to generate parametrized reports.
+  - input_files:
+      type: path
+      description: One or multiple files serving as input data for the notebook.
+      pattern: "*"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - report:
+      type: file
+      description: HTML report generated from Rmarkdown
+      pattern: "*.html"
+  - session_info:
+      type: file
+      description: dump of R SessionInfo
+      pattern: "*.log"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@grst"

modules/rmarkdownnotebook/parametrize.nf

@@ -0,0 +1,36 @@
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.DumperOptions
+
+
+/**
+ * Multiline code blocks need to have the same indentation level
+ * as the `script:` section. This function re-indents code to the specified level.
+ */
+def indent_code_block(code, n_spaces) {
+    def indent_str = " ".multiply(n_spaces)
+    return code.stripIndent().split("\n").join("\n" + indent_str)
+}
+
+/**
+ * Create a config YAML file from a groovy map
+ *
+ * @params task The process' `task` variable
+ * @returns a line to be inserted in the bash script.
+ */
+def dump_params_yml(params) {
+    DumperOptions options = new DumperOptions();
+    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+    def yaml = new Yaml(options)
+    def yaml_str = yaml.dump(params)
+
+    // Writing the .params.yml file directly as follows does not work.
+    // It only works in 'exec:', but not if there is a `script:` section:
+    // task.workDir.resolve('.params.yml').text = yaml_str
+
+    // Therefore, we inject it into the bash script:
+    return """\
+        cat <<"END_PARAMS_SECTION" > ./.params.yml
+        ${indent_code_block(yaml_str, 8)}
+        END_PARAMS_SECTION
+    """
+}

tests/config/pytest_modules.yml

@@ -585,6 +585,10 @@ ivar/variants:
   - modules/ivar/variants/**
   - tests/modules/ivar/variants/**
 
+jupyternotebook:
+  - modules/jupyternotebook/**
+  - tests/modules/jupyternotebook/**
+
 kallisto/index:
   - modules/kallisto/index/**
   - tests/modules/kallisto/index/**
@@ -871,6 +875,10 @@ raxmlng:
   - modules/raxmlng/**
   - tests/modules/raxmlng/**
 
+rmarkdownnotebook:
+  - modules/rmarkdownnotebook/**
+  - tests/modules/rmarkdownnotebook/**
+
 roary:
   - modules/roary/**
   - tests/modules/roary/**

tests/config/test_data.config

@@ -229,5 +229,15 @@ params {
                 filelist                                      = "${test_data_dir}/genomics/homo_sapiens/pacbio/txt/filelist.txt"
             }
         }
+        'generic' {
+            'notebooks' {
+                rmarkdown = "${test_data_dir}/generic/notebooks/rmarkdown/rmarkdown_notebook.Rmd"
+                ipython_md = "${test_data_dir}/generic/notebooks/jupyter/ipython_notebook.md"
+                ipython_ipynb = "${test_data_dir}/generic/notebooks/jupyter/ipython_notebook.ipynb"
+            }
+            'txt' {
+                hello = "${test_data_dir}/generic/txt/hello.txt"
+            }
+        }
     }
 }

tests/modules/jupyternotebook/main.nf

@@ -0,0 +1,49 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { JUPYTERNOTEBOOK } from '../../../modules/jupyternotebook/main.nf' addParams(
+    parametrize: false, options: [:]
+)
+include { JUPYTERNOTEBOOK as JUPYTERNOTEBOOK_PARAMETRIZE } from '../../../modules/jupyternotebook/main.nf' addParams(
+    options: [:]
+)
+include { JUPYTERNOTEBOOK as JUPYTERNOTEBOOK_PARAMETRIZE_IPYNB } from '../../../modules/jupyternotebook/main.nf' addParams(
+    options: [:]
+)
+
+workflow test_jupyternotebook {
+
+    input = [ [ id:'test_jupyter' ], // meta map
+              file(params.test_data['generic']['notebooks']['ipython_md'], checkIfExists: true) ]
+
+    JUPYTERNOTEBOOK ( input, [:], [])
+
+}
+
+workflow test_jupyternotebook_parametrize {
+
+    input = [ [ id:'test_jupyter' ], // meta map
+              file(params.test_data['generic']['notebooks']['ipython_md'], checkIfExists: true) ]
+
+    JUPYTERNOTEBOOK_PARAMETRIZE(
+        input,
+        [input_filename: "hello.txt", n_iter: 12],
+        file(params.test_data['generic']['txt']['hello'], checkIfExists: true)
+    )
+
+}
+
+workflow test_jupyternotebook_parametrize_ipynb {
+
+    input = [ [ id:'test_jupyter' ], // meta map
+              file(params.test_data['generic']['notebooks']['ipython_ipynb'], checkIfExists: true) ]
+
+    JUPYTERNOTEBOOK_PARAMETRIZE_IPYNB(
+        input,
+        [input_filename: "hello.txt", n_iter: 12],
+        file(params.test_data['generic']['txt']['hello'], checkIfExists: true)
+    )
+
+}
+

tests/modules/jupyternotebook/test.yml

@@ -0,0 +1,30 @@
+- name: jupyternotebook test_jupyternotebook
+  command: nextflow run tests/modules/jupyternotebook -entry test_jupyternotebook -c tests/config/nextflow.config
+  tags:
+    - jupyternotebook
+  files:
+    - path: output/jupyternotebook/test_jupyter.html
+      contains:
+        - "n_iter = 10"
+
+- name: jupyternotebook test_jupyternotebook_parametrize
+  command: nextflow run tests/modules/jupyternotebook -entry test_jupyternotebook_parametrize -c tests/config/nextflow.config
+  tags:
+    - jupyternotebook
+  files:
+    - path: output/jupyternotebook/artifacts/artifact.txt
+      md5sum: 8ddd8be4b179a529afa5f2ffae4b9858
+    - path: output/jupyternotebook/test_jupyter.html
+      contains:
+        - "n_iter = 12"
+
+- name: jupyternotebook test_jupyternotebook_parametrize_ipynb
+  command: nextflow run tests/modules/jupyternotebook -entry test_jupyternotebook_parametrize_ipynb -c tests/config/nextflow.config
+  tags:
+    - jupyternotebook
+  files:
+    - path: output/jupyternotebook/artifacts/artifact.txt
+      md5sum: 8ddd8be4b179a529afa5f2ffae4b9858
+    - path: output/jupyternotebook/test_jupyter.html
+      contains:
+        - "n_iter = 12"

tests/modules/rmarkdownnotebook/main.nf

@@ -0,0 +1,33 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { RMARKDOWNNOTEBOOK } from '../../../modules/rmarkdownnotebook/main.nf' addParams(
+    parametrize: false, options: [:]
+)
+include { RMARKDOWNNOTEBOOK as RMARKDOWNNOTEBOOK_PARAMETRIZE } from '../../../modules/rmarkdownnotebook/main.nf' addParams(
+    options: [:]
+)
+
+workflow test_rmarkdown {
+
+    input = [ [ id:'test_rmd' ], // meta map
+              file(params.test_data['generic']['notebooks']['rmarkdown'], checkIfExists: true) ]
+
+    RMARKDOWNNOTEBOOK ( input, [:], [])
+
+}
+
+workflow test_rmarkdown_parametrize {
+
+    input = [ [ id:'test_rmd' ], // meta map
+              file(params.test_data['generic']['notebooks']['rmarkdown'], checkIfExists: true) ]
+
+    RMARKDOWNNOTEBOOK_PARAMETRIZE(
+        input,
+        [input_filename: "hello.txt", n_iter: 12],
+        file(params.test_data['generic']['txt']['hello'], checkIfExists: true)
+    )
+
+}
+

tests/modules/rmarkdownnotebook/test.yml

@@ -0,0 +1,27 @@
+- name: rmarkdownnotebook test_rmarkdown
+  command: nextflow run tests/modules/rmarkdownnotebook -entry test_rmarkdown -c tests/config/nextflow.config
+  tags:
+    - rmarkdownnotebook
+  files:
+    - path: output/rmarkdownnotebook/session_info.log
+      contains:
+        - R version 4.1.0
+        - yaml_2.2.1
+    - path: output/rmarkdownnotebook/test_rmd.html
+      contains:
+        - "n_iter = 10"
+
+- name: rmarkdownnotebook test_rmarkdown_parametrize
+  command: nextflow run tests/modules/rmarkdownnotebook -entry test_rmarkdown_parametrize -c tests/config/nextflow.config
+  tags:
+    - rmarkdownnotebook
+  files:
+    - path: output/rmarkdownnotebook/artifacts/artifact.txt
+      md5sum: b10a8db164e0754105b7a99be72e3fe5
+    - path: output/rmarkdownnotebook/session_info.log
+      contains:
+        - R version 4.1.0
+        - yaml_2.2.1
+    - path: output/rmarkdownnotebook/test_rmd.html
+      contains:
+        - "n_iter = 12"