Skip to content

Commit

Permalink
Merge pull request #3128 from mirpedrol/template-customisation-nfcore…
Browse files Browse the repository at this point in the history
…modules

Template: add option to exclude nf-core components from pipeline template
  • Loading branch information
mirpedrol authored Aug 28, 2024
2 parents b4acddc + 8351e0c commit 4e385ab
Show file tree
Hide file tree
Showing 10 changed files with 377 additions and 311 deletions.
5 changes: 4 additions & 1 deletion nf_core/commands_pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,12 @@ def pipelines_lint(
ctx.obj["hide_progress"],
)
swf_failed = 0
module_failed = 0
if subworkflow_lint_obj is not None:
swf_failed = len(subworkflow_lint_obj.failed)
if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0:
if module_lint_obj is not None:
module_failed = len(module_lint_obj.failed)
if len(lint_obj.failed) + module_failed + swf_failed > 0:
sys.exit(1)
except AssertionError as e:
log.critical(e)
Expand Down
2 changes: 2 additions & 0 deletions nf_core/pipeline-template/.editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ indent_style = space
[*.{md,yml,yaml,html,css,scss,js}]
indent_size = 2

{%- if modules %}
# These files are edited and tested upstream in nf-core/modules
[/modules/nf-core/**]
charset = unset
Expand All @@ -24,6 +25,7 @@ end_of_line = unset
insert_final_newline = unset
trim_trailing_whitespace = unset
indent_style = unset
{%- endif %}

{%- if email %}
[/assets/email*]
Expand Down
16 changes: 12 additions & 4 deletions nf_core/pipeline-template/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/

include { {{ short_name|upper }} } from './workflows/{{ short_name }}'
{%- if modules %}
include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'
include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'
{%- if igenomes %}
Expand All @@ -33,7 +34,7 @@ include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_{{ s
// This is an example of how to use getGenomeAttribute() to fetch parameters
// from igenomes.config using `--genome`
params.fasta = getGenomeAttribute('fasta')
{% endif %}
{% endif %}{% endif %}
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
NAMED WORKFLOWS FOR PIPELINE
Expand All @@ -56,10 +57,10 @@ workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} {
{{ short_name|upper }} (
samplesheet
)
{%- if multiqc %}
{%- if multiqc %}{%- if modules %}
emit:
multiqc_report = {{ short_name|upper }}.out.multiqc_report // channel: /path/to/multiqc_report.html
{%- endif %}
{%- endif %}{%- endif %}
}
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand All @@ -71,6 +72,7 @@ workflow {

main:

{%- if modules %}
//
// SUBWORKFLOW: Run initialisation tasks
//
Expand All @@ -82,14 +84,19 @@ workflow {
params.outdir,
params.input
)

{% endif %}
//
// WORKFLOW: Run main workflow
//
{{ prefix_nodash|upper }}_{{ short_name|upper }} (
{%- if modules %}
PIPELINE_INITIALISATION.out.samplesheet
{%- else %}
params.input
{%- endif %}
)

{%- if modules %}
//
// SUBWORKFLOW: Run completion tasks
//
Expand All @@ -104,6 +111,7 @@ workflow {
{% if adaptivecard or slackreport %}params.hook_url,{% endif %}
{% if multiqc %}{{ prefix_nodash|upper }}_{{ short_name|upper }}.out.multiqc_report{% endif %}
)
{%- endif %}
}

/*
Expand Down
19 changes: 15 additions & 4 deletions nf_core/pipeline-template/nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,21 @@ params {
validate_params = true

}

{% if modules %}
// Load base.config by default for all pipelines
includeConfig 'conf/base.config'
{%- else %}
process {
// TODO nf-core: Check the defaults for all processes
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
time = { check_max( 4.h * task.attempt, 'time' ) }

errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
maxRetries = 1
maxErrors = '-1'
}
{% endif %}
{% if nf_core_configs -%}
// Load nf-core custom profiles from different Institutions
try {
Expand Down Expand Up @@ -289,7 +300,7 @@ validation {
-\033[2m----------------------------------------------------\033[0m-
"""
afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""}
* The nf-core framework
* The nf-core framework
https://doi.org/10.1038/s41587-020-0439-x
* Software dependencies
Expand All @@ -302,10 +313,10 @@ validation {
}{% endif %}
}
{% endif -%}

{%- if modules %}
// Load modules.config for DSL2 module specific options
includeConfig 'conf/modules.config'

{% endif %}
// Function to ensure that resource requirements don't go beyond
// a maximum limit
def check_max(obj, type) {
Expand Down
2 changes: 2 additions & 0 deletions nf_core/pipeline-template/nextflow_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"fa_icon": "fas fa-book",
"help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details."
},
{%- if modules %}
"fasta": {
"type": "string",
"format": "file-path",
Expand All @@ -66,6 +67,7 @@
"help_text": "This parameter is *mandatory* if `--genome` is not specified. If you don't have a BWA index available this will be generated for you automatically. Combine with `--save_reference` to save BWA index for future runs.",
"fa_icon": "far fa-file-code"
},
{%- endif %}
"igenomes_ignore": {
"type": "boolean",
"description": "Do not load the iGenomes reference config.",
Expand Down
4 changes: 4 additions & 0 deletions nf_core/pipeline-template/workflows/pipeline.nf
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/

{%- if modules %}
{% if fastqc %}include { FASTQC } from '../modules/nf-core/fastqc/main'{% endif %}
{% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %}
{% if nf_schema %}include { paramsSummaryMap } from 'plugin/nf-schema'{% endif %}
{% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %}
include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline'
{% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %}
{%- endif %}

/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand All @@ -22,6 +24,7 @@ workflow {{ short_name|upper }} {
take:
ch_samplesheet // channel: samplesheet read in from --input

{%- if modules %}
main:

ch_versions = Channel.empty()
Expand Down Expand Up @@ -98,6 +101,7 @@ workflow {{ short_name|upper }} {
emit:
{%- if multiqc %}multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html{% endif %}
versions = ch_versions // channel: [ path(versions.yml) ]
{% endif %}
}

/*
Expand Down
23 changes: 23 additions & 0 deletions nf_core/pipelines/create/templatefeatures.yml
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,29 @@ fastqc:
The pipeline will include the FastQC module.
nfcore_pipelines: True
custom_pipelines: True
modules:
skippable_paths:
- "conf/base.config"
- "conf/modules.config"
- "modules.json"
- "modules"
- "subworkflows"
short_description: "Use nf-core components"
description: "Include all required files to use nf-core modules and subworkflows"
help_text: |
It is *recommended* to use this feature if you want to use modules and subworkflows in your pipeline.
This will add all required files to use nf-core components or any compatible components from private repos by using `nf-core modules` and `nf-core subworkflows` commands.
linting:
nfcore_components: False
modules_json: False
base_config: False
modules_config: False
files_exist:
- "conf/base.config"
- "conf/modules.config"
- "modules.json"
nfcore_pipelines: False
custom_pipelines: True
changelog:
skippable_paths:
- "CHANGELOG.md"
Expand Down
93 changes: 50 additions & 43 deletions nf_core/pipelines/lint/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import logging
import os
from pathlib import Path
from typing import List, Tuple, Union
from typing import List, Optional, Tuple, Union

import git
import rich
Expand Down Expand Up @@ -177,7 +177,8 @@ def _load_lint_config(self) -> bool:
# Check if we have any keys that don't match lint test names
if self.lint_config is not None:
for k in self.lint_config:
if k not in self.lint_tests:
if k != "nfcore_components" and k not in self.lint_tests:
# nfcore_components is an exception to allow custom pipelines without nf-core components
log.warning(f"Found unrecognised test name '{k}' in pipeline lint config")
is_correct = False

Expand Down Expand Up @@ -546,7 +547,7 @@ def run_linting(
md_fn=None,
json_fn=None,
hide_progress: bool = False,
) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]:
) -> Tuple[PipelineLint, Optional[ComponentLint], Optional[ComponentLint]]:
"""Runs all nf-core linting checks on a given Nextflow pipeline project
in either `release` mode or `normal` mode (default). Returns an object
of type :class:`PipelineLint` after finished.
Expand Down Expand Up @@ -591,41 +592,45 @@ def run_linting(
lint_obj._load_lint_config()
lint_obj.load_pipeline_config()

# Create the modules lint object
module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress)
# Create the subworkflows lint object
try:
subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress)
except LookupError:
if "nfcore_components" in lint_obj.lint_config and not lint_obj.lint_config["nfcore_components"]:
module_lint_obj = None
subworkflow_lint_obj = None

# Verify that the pipeline is correctly configured and has a modules.json file
module_lint_obj.has_valid_directory()
module_lint_obj.has_modules_file()
# Run only the tests we want
if key:
# Select only the module lint tests
module_lint_tests = list(
set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)))
)
# Select only the subworkflow lint tests
subworkflow_lint_tests = list(
set(key).intersection(
set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True))
)
)
else:
# If no key is supplied, run the default modules tests
module_lint_tests = list(("module_changes", "module_version"))
subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version"))
module_lint_obj.filter_tests_by_key(module_lint_tests)
if subworkflow_lint_obj is not None:
subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests)

# Set up files for component linting test
module_lint_obj.set_up_pipeline_files()
if subworkflow_lint_obj is not None:
subworkflow_lint_obj.set_up_pipeline_files()
# Create the modules lint object
module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress)
# Create the subworkflows lint object
try:
subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress)
except LookupError:
subworkflow_lint_obj = None

# Verify that the pipeline is correctly configured and has a modules.json file
module_lint_obj.has_valid_directory()
module_lint_obj.has_modules_file()
# Run only the tests we want
if key:
# Select only the module lint tests
module_lint_tests = list(
set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)))
)
# Select only the subworkflow lint tests
subworkflow_lint_tests = list(
set(key).intersection(
set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True))
)
)
else:
# If no key is supplied, run the default modules tests
module_lint_tests = list(("module_changes", "module_version"))
subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version"))
module_lint_obj.filter_tests_by_key(module_lint_tests)
if subworkflow_lint_obj is not None:
subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests)

# Set up files for component linting test
module_lint_obj.set_up_pipeline_files()
if subworkflow_lint_obj is not None:
subworkflow_lint_obj.set_up_pipeline_files()

# Run the pipeline linting tests
try:
Expand All @@ -635,21 +640,23 @@ def run_linting(
log.info("Stopping tests...")
return lint_obj, module_lint_obj, subworkflow_lint_obj

# Run the module lint tests
if len(module_lint_obj.all_local_components) > 0:
module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True)
if len(module_lint_obj.all_remote_components) > 0:
module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False)
# Run the subworkflows lint tests
if module_lint_obj is not None:
# Run the module lint tests
if len(module_lint_obj.all_local_components) > 0:
module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True)
if len(module_lint_obj.all_remote_components) > 0:
module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False)
if subworkflow_lint_obj is not None:
# Run the subworkflows lint tests
if len(subworkflow_lint_obj.all_local_components) > 0:
subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True)
if len(subworkflow_lint_obj.all_remote_components) > 0:
subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False)

# Print the results
lint_obj._print_results(show_passed)
module_lint_obj._print_results(show_passed, sort_by=sort_by)
if module_lint_obj is not None:
module_lint_obj._print_results(show_passed, sort_by=sort_by)
if subworkflow_lint_obj is not None:
subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by)
nf_core.pipelines.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj)
Expand Down
13 changes: 10 additions & 3 deletions nf_core/pipelines/lint_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,22 @@ def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj):
swf_passed = 0
swf_warned = 0
swf_failed = 0
module_passed = 0
module_warned = 0
module_failed = 0
if subworkflow_lint_obj is not None:
swf_passed = len(subworkflow_lint_obj.passed)
swf_warned = len(subworkflow_lint_obj.warned)
swf_failed = len(subworkflow_lint_obj.failed)
nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) + swf_passed
if module_lint_obj is not None:
module_passed = len(module_lint_obj.passed)
module_warned = len(module_lint_obj.warned)
module_failed = len(module_lint_obj.failed)
nbr_passed = len(lint_obj.passed) + module_passed + swf_passed
nbr_ignored = len(lint_obj.ignored)
nbr_fixed = len(lint_obj.fixed)
nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) + swf_warned
nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed
nbr_warned = len(lint_obj.warned) + module_warned + swf_warned
nbr_failed = len(lint_obj.failed) + module_failed + swf_failed

summary_colour = "red" if nbr_failed > 0 else "green"
table = Table(box=rich.box.ROUNDED, style=summary_colour)
Expand Down
Loading

0 comments on commit 4e385ab

Please sign in to comment.