From 0f9262c6b52ec10d9d40c2aefac383be31a4a90b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 23 Mar 2024 00:34:54 +0100 Subject: [PATCH 01/12] Add plugin entrypoints for toolong See Textualize/toolong#47 --- nf_core/toolong_formatter.py | 203 +++++++++++++++++++++++++++++++++++ setup.py | 6 ++ 2 files changed, 209 insertions(+) create mode 100644 nf_core/toolong_formatter.py diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py new file mode 100644 index 0000000000..dd973b2fe8 --- /dev/null +++ b/nf_core/toolong_formatter.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import re +from datetime import datetime +from typing import Optional + +from rich.text import Text +from toolong import timestamps +from toolong.highlighter import LogHighlighter +from typing_extensions import TypeAlias + +ParseResult: TypeAlias = "tuple[Optional[datetime], str, Text]" + + +class LogFormat: + def parse(self, line: str) -> ParseResult | None: + raise NotImplementedError() + + +class NextflowRegexLogFormatOne(LogFormat): + REGEX = re.compile(".*?") + LOG_LEVELS = { + "DEBUG": ["dim white on black", ""], + "INFO": ["bold black on green", "on #042C07"], + "WARN": ["bold black on yellow", "on #44450E"], + "ERROR": ["bold black on red", "on #470005"], + } + + highlighter = LogHighlighter() + + def parse(self, line: str) -> ParseResult | None: + match = self.REGEX.fullmatch(line) + if match is None: + return None + + text = Text.from_ansi(line) + groups = match.groupdict() + if date := groups.get("date", None): + _, timestamp = timestamps.parse(groups["date"]) + text.highlight_words([date], "not bold magenta") + if thread := groups.get("thread", None): + text.highlight_words([thread], "blue") + if log_level := groups.get("log_level", None): + text.highlight_words([f" {log_level} "], self.LOG_LEVELS[log_level][0]) + text.stylize_before(self.LOG_LEVELS[log_level][1]) + if logger_name := groups.get("logger_name", None): + text.highlight_words([logger_name], "cyan") + if process_name := groups.get("process_name", None): + text.highlight_words([process_name], "bold cyan") + if message := groups.get("message", None): + text.highlight_words([message], "dim" if log_level == "DEBUG" else "") + + return None, line, text + + +class NextflowRegexLogFormatTwo(LogFormat): + REGEX = re.compile(".*?") + highlighter = LogHighlighter() + + def parse(self, line: str) -> ParseResult | None: + match = self.REGEX.fullmatch(line) + if match is None: + return None + + text = Text.from_ansi(line) + text.stylize_before("dim") + groups = match.groupdict() + if process := groups.get("process", None): + text.highlight_words([process], "blue not dim") + if process_name := groups.get("process_name", None): + text.highlight_words([process_name], "bold cyan not dim") + + return None, line, text + + +class NextflowRegexLogFormatThree(LogFormat): + REGEX = re.compile(".*?") + CHANNEL_TYPES = { + "(value)": "green", + "(cntrl)": "yellow", + "(queue)": "magenta", + } + highlighter = LogHighlighter() + + def parse(self, line: str) -> ParseResult | None: + match = self.REGEX.fullmatch(line) + if match is None: + return None + + text = Text.from_ansi(line) + groups = match.groupdict() + if port := groups.get("port", None): + text.highlight_words([port], "blue") + if channel_type := groups.get("channel_type", None): + text.highlight_words([channel_type], self.CHANNEL_TYPES[channel_type]) + if channel_state := groups.get("channel_state", None): + text.highlight_words([channel_state], "cyan" if channel_state == "OPEN" else "yellow") + text.highlight_words(["; channel:"], "dim") + if channel_name := groups.get("channel_name", None): + text.highlight_words([channel_name], "cyan") + + return None, line, text + + +class NextflowRegexLogFormatFour(LogFormat): + REGEX = re.compile(".*?") + highlighter = LogHighlighter() + + def parse(self, line: str) -> ParseResult | None: + match = self.REGEX.fullmatch(line) + if match is None: + return None + + text = Text.from_ansi(line) + text.stylize_before("dim") + groups = match.groupdict() + text.highlight_words(["status="], "dim") + if status := groups.get("status", None): + text.highlight_words([status], "cyan not dim") + + return None, line, text + + +class NextflowRegexLogFormatFive(LogFormat): + REGEX = re.compile(".*?") + highlighter = LogHighlighter() + + def parse(self, line: str) -> ParseResult | None: + match = self.REGEX.fullmatch(line) + if match is None: + return None + + text = Text.from_ansi(line) + text.stylize_before("dim") + groups = match.groupdict() + if script_id := groups.get("script_id", None): + text.highlight_words([script_id], "blue") + if script_path := groups.get("script_path", None): + text.highlight_words([script_path], "magenta") + + return None, line, text + + +class NextflowLogFormat(NextflowRegexLogFormatOne): + REGEX = re.compile( + r"(?P\w+-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) (?P\[.*\]?) (?P\w+)\s+(?P[\w\.]+) - (?P.*?)$" + ) + + +class NextflowLogFormatActiveProcess(NextflowRegexLogFormatTwo): + REGEX = re.compile(r"^(?P\[process\]) (?P.*?)(?P[^:]+?)?$") + + +class NextflowLogFormatActiveProcessDetails(NextflowRegexLogFormatThree): + REGEX = re.compile( + r" (?Pport \d+): (?P\((value|queue|cntrl)\)) (?P\S+)\s+; channel: (?P.*?)$" + ) + + +class NextflowLogFormatActiveProcessStatus(NextflowRegexLogFormatFour): + REGEX = re.compile(r"^ status=(?P.*?)?$") + + +class NextflowLogFormatScriptParse(NextflowRegexLogFormatFive): + REGEX = re.compile(r"^ (?PScript_\w+:) (?P.*?)$") + + +def nextflow_formatters(formats): + return [ + NextflowLogFormat(), + NextflowLogFormatActiveProcess(), + NextflowLogFormatActiveProcessDetails(), + NextflowLogFormatActiveProcessStatus(), + NextflowLogFormatScriptParse(), + ] + + +def nextflow_format_parser(format_parser): + class FormatParser(format_parser): + """Parses a log line.""" + + def __init__(self) -> None: + super().__init__() + self._log_status = "" + + def parse(self, line: str) -> ParseResult: + """Parse a line.""" + + for logtype in ["DEBUG", "INFO", "WARN", "ERROR"]: + if logtype in line: + self._log_status = logtype + return super().parse(line) + text = Text(line) + if text.plain == text.markup: + if self._log_status == "DEBUG": + text.stylize("dim") + if self._log_status == "WARN": + text.stylize("yellow") + if self._log_status == "ERROR": + text.stylize("red") + return None, line, text + + return FormatParser diff --git a/setup.py b/setup.py index fc7b69ac1e..4e53f4ca56 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,12 @@ entry_points={ "console_scripts": ["nf-core=nf_core.__main__:run_nf_core"], "refgenie.hooks.post_update": ["nf-core-refgenie=nf_core.refgenie:update_config"], + "toolong.application.formats": [ + "nextflow_formatters = nf_core.toolong_formatter:nextflow_formatters", + ], + "toolong.application.format_parsers": [ + "nextflow_format_parser = nf_core.toolong_formatter:nextflow_format_parser", + ], }, python_requires=">=3.8, <4", install_requires=required, From 9dd94c58f0830eac356804e4364ab12c68a04ecd Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 22 Mar 2024 23:37:46 +0000 Subject: [PATCH 02/12] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..7441566a62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Add plugin entrypoints for toolong ([#2895](https://github.com/nf-core/tools/pull/2895)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From f3361b3fbaba2dc08dd4193dd4ba6139660e09cf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 23 Mar 2024 22:31:11 +0100 Subject: [PATCH 03/12] Nicer formatting - no background colour, gutter for log type --- nf_core/toolong_formatter.py | 56 ++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index dd973b2fe8..5880ee110c 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -11,6 +11,14 @@ ParseResult: TypeAlias = "tuple[Optional[datetime], str, Text]" +MOVE_LOG_LEVEL_COL = True +LOG_LEVELS = { + "DEBUG": ["dim white on black", "dim"], + "INFO": ["bold black on green", ""], + "WARN": ["bold black on yellow", "yellow"], + "ERROR": ["bold black on red", "red"], +} + class LogFormat: def parse(self, line: str) -> ParseResult | None: @@ -19,12 +27,6 @@ def parse(self, line: str) -> ParseResult | None: class NextflowRegexLogFormatOne(LogFormat): REGEX = re.compile(".*?") - LOG_LEVELS = { - "DEBUG": ["dim white on black", ""], - "INFO": ["bold black on green", "on #042C07"], - "WARN": ["bold black on yellow", "on #44450E"], - "ERROR": ["bold black on red", "on #470005"], - } highlighter = LogHighlighter() @@ -41,8 +43,7 @@ def parse(self, line: str) -> ParseResult | None: if thread := groups.get("thread", None): text.highlight_words([thread], "blue") if log_level := groups.get("log_level", None): - text.highlight_words([f" {log_level} "], self.LOG_LEVELS[log_level][0]) - text.stylize_before(self.LOG_LEVELS[log_level][1]) + text.highlight_words([f" {log_level} "], LOG_LEVELS[log_level][0]) if logger_name := groups.get("logger_name", None): text.highlight_words([logger_name], "cyan") if process_name := groups.get("process_name", None): @@ -186,18 +187,37 @@ def __init__(self) -> None: def parse(self, line: str) -> ParseResult: """Parse a line.""" - for logtype in ["DEBUG", "INFO", "WARN", "ERROR"]: + # Use the toolong parser with custom formatters + _, line, text = super().parse(line) + + # Custom formatting with log levels + for logtype in LOG_LEVELS.keys(): if logtype in line: + # Set log status for next lines, if multi-line self._log_status = logtype - return super().parse(line) + # Set the base stlying for this line + text.stylize_before(LOG_LEVELS[logtype][1]) + # Move the "INFO" log level to the start of the line + if MOVE_LOG_LEVEL_COL: + line = "{} {}".format( + logtype, + line.replace(f" {logtype} ", ""), + ) + logtype_str = f"[{LOG_LEVELS[logtype][0]}] {logtype: <5} [/] " + text = Text.from_markup( + logtype_str + text.markup.replace(f" {logtype} ", "[reset] [/]"), + ) + # Return - on to next line + return _, line, text + + # Multi-line log message + # Strip automatic formatting, which does weird stuff text = Text(line) - if text.plain == text.markup: - if self._log_status == "DEBUG": - text.stylize("dim") - if self._log_status == "WARN": - text.stylize("yellow") - if self._log_status == "ERROR": - text.stylize("red") - return None, line, text + for logtype in LOG_LEVELS.keys(): + if self._log_status == logtype: + text = Text.from_markup(f"[{LOG_LEVELS[logtype][0]}] [/] " + text.markup) + text.stylize_before(LOG_LEVELS[logtype][1]) + + return _, line, text return FormatParser From 4a9a3a8f5870202e4b3842f20f9f89dee66fc31f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 23 Mar 2024 22:55:57 +0100 Subject: [PATCH 04/12] Clean up code --- nf_core/toolong_formatter.py | 56 ++++++++++++------------------------ 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index 5880ee110c..2fef05bbe1 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -25,8 +25,10 @@ def parse(self, line: str) -> ParseResult | None: raise NotImplementedError() -class NextflowRegexLogFormatOne(LogFormat): - REGEX = re.compile(".*?") +class NextflowLogFormat(LogFormat): + REGEX = re.compile( + r"(?P\w+-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) (?P\[.*\]?) (?P\w+)\s+(?P[\w\.]+) - (?P.*?)$" + ) highlighter = LogHighlighter() @@ -51,11 +53,11 @@ def parse(self, line: str) -> ParseResult | None: if message := groups.get("message", None): text.highlight_words([message], "dim" if log_level == "DEBUG" else "") - return None, line, text + return timestamp, line, text -class NextflowRegexLogFormatTwo(LogFormat): - REGEX = re.compile(".*?") +class NextflowLogFormatActiveProcess(LogFormat): + REGEX = re.compile(r"^(?P\[process\]) (?P.*?)(?P[^:]+?)?$") highlighter = LogHighlighter() def parse(self, line: str) -> ParseResult | None: @@ -74,8 +76,10 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowRegexLogFormatThree(LogFormat): - REGEX = re.compile(".*?") +class NextflowLogFormatActiveProcessDetails(LogFormat): + REGEX = re.compile( + r" (?Pport \d+): (?P\((value|queue|cntrl)\)) (?P\S+)\s+; channel: (?P.*?)$" + ) CHANNEL_TYPES = { "(value)": "green", "(cntrl)": "yellow", @@ -103,8 +107,8 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowRegexLogFormatFour(LogFormat): - REGEX = re.compile(".*?") +class NextflowLogFormatActiveProcessStatus(LogFormat): + REGEX = re.compile(r"^ status=(?P.*?)?$") highlighter = LogHighlighter() def parse(self, line: str) -> ParseResult | None: @@ -122,8 +126,8 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowRegexLogFormatFive(LogFormat): - REGEX = re.compile(".*?") +class NextflowLogFormatScriptParse(LogFormat): + REGEX = re.compile(r"^ (?PScript_\w+:) (?P.*?)$") highlighter = LogHighlighter() def parse(self, line: str) -> ParseResult | None: @@ -142,30 +146,6 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowLogFormat(NextflowRegexLogFormatOne): - REGEX = re.compile( - r"(?P\w+-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) (?P\[.*\]?) (?P\w+)\s+(?P[\w\.]+) - (?P.*?)$" - ) - - -class NextflowLogFormatActiveProcess(NextflowRegexLogFormatTwo): - REGEX = re.compile(r"^(?P\[process\]) (?P.*?)(?P[^:]+?)?$") - - -class NextflowLogFormatActiveProcessDetails(NextflowRegexLogFormatThree): - REGEX = re.compile( - r" (?Pport \d+): (?P\((value|queue|cntrl)\)) (?P\S+)\s+; channel: (?P.*?)$" - ) - - -class NextflowLogFormatActiveProcessStatus(NextflowRegexLogFormatFour): - REGEX = re.compile(r"^ status=(?P.*?)?$") - - -class NextflowLogFormatScriptParse(NextflowRegexLogFormatFive): - REGEX = re.compile(r"^ (?PScript_\w+:) (?P.*?)$") - - def nextflow_formatters(formats): return [ NextflowLogFormat(), @@ -188,7 +168,7 @@ def parse(self, line: str) -> ParseResult: """Parse a line.""" # Use the toolong parser with custom formatters - _, line, text = super().parse(line) + timestamp, line, text = super().parse(line) # Custom formatting with log levels for logtype in LOG_LEVELS.keys(): @@ -208,7 +188,7 @@ def parse(self, line: str) -> ParseResult: logtype_str + text.markup.replace(f" {logtype} ", "[reset] [/]"), ) # Return - on to next line - return _, line, text + return timestamp, line, text # Multi-line log message # Strip automatic formatting, which does weird stuff @@ -218,6 +198,6 @@ def parse(self, line: str) -> ParseResult: text = Text.from_markup(f"[{LOG_LEVELS[logtype][0]}] [/] " + text.markup) text.stylize_before(LOG_LEVELS[logtype][1]) - return _, line, text + return timestamp, line, text return FormatParser From 821fb74731a98094a73e5324d700fdd7bf7767a9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 23 Mar 2024 23:45:33 +0100 Subject: [PATCH 05/12] Only enable Nextflow formatting if a .nextflow.log file --- nf_core/toolong_formatter.py | 38 +++++++++++++++++++++++++++--------- setup.py | 3 +++ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index 2fef05bbe1..ab35a1d4eb 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -5,12 +5,12 @@ from typing import Optional from rich.text import Text -from toolong import timestamps from toolong.highlighter import LogHighlighter from typing_extensions import TypeAlias ParseResult: TypeAlias = "tuple[Optional[datetime], str, Text]" + MOVE_LOG_LEVEL_COL = True LOG_LEVELS = { "DEBUG": ["dim white on black", "dim"], @@ -19,6 +19,16 @@ "ERROR": ["bold black on red", "red"], } +IS_NEXTFLOW = False + + +def nf_toolong_on_init(scan): + import nf_core.toolong_formatter + + for file_path in scan.file_paths: + if file_path.startswith(".nextflow.log"): + nf_core.toolong_formatter.IS_NEXTFLOW = True + class LogFormat: def parse(self, line: str) -> ParseResult | None: @@ -40,7 +50,7 @@ def parse(self, line: str) -> ParseResult | None: text = Text.from_ansi(line) groups = match.groupdict() if date := groups.get("date", None): - _, timestamp = timestamps.parse(groups["date"]) + timestamp = datetime.strptime(groups["date"], "%b-%d %H:%M:%S.%f") text.highlight_words([date], "not bold magenta") if thread := groups.get("thread", None): text.highlight_words([thread], "blue") @@ -147,16 +157,22 @@ def parse(self, line: str) -> ParseResult | None: def nextflow_formatters(formats): - return [ - NextflowLogFormat(), - NextflowLogFormatActiveProcess(), - NextflowLogFormatActiveProcessDetails(), - NextflowLogFormatActiveProcessStatus(), - NextflowLogFormatScriptParse(), - ] + import nf_core.toolong_formatter + + if nf_core.toolong_formatter.IS_NEXTFLOW: + return [ + NextflowLogFormat(), + NextflowLogFormatActiveProcess(), + NextflowLogFormatActiveProcessDetails(), + NextflowLogFormatActiveProcessStatus(), + NextflowLogFormatScriptParse(), + ] + return formats def nextflow_format_parser(format_parser): + import nf_core.toolong_formatter + class FormatParser(format_parser): """Parses a log line.""" @@ -170,6 +186,10 @@ def parse(self, line: str) -> ParseResult: # Use the toolong parser with custom formatters timestamp, line, text = super().parse(line) + # Return if not a netflow log file + if not nf_core.toolong_formatter.IS_NEXTFLOW: + return timestamp, line, text + # Custom formatting with log levels for logtype in LOG_LEVELS.keys(): if logtype in line: diff --git a/setup.py b/setup.py index 4e53f4ca56..b7ed6ea1f2 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,9 @@ "toolong.application.format_parsers": [ "nextflow_format_parser = nf_core.toolong_formatter:nextflow_format_parser", ], + "toolong.application.on_init": [ + "nf_toolong_on_init = nf_core.toolong_formatter:nf_toolong_on_init", + ], }, python_requires=">=3.8, <4", install_requires=required, From c82861d2ddfe31f1824c3d3c4146d77ce8da6bcd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 24 Mar 2024 00:21:41 +0100 Subject: [PATCH 06/12] Add to requirements.txt Temporary github URL, until PR is merged --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6b5b3ab57d..25df1a95d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,6 @@ rich-click>=1.6.1 rich>=13.3.1 tabulate trogon +# Remove dev brach dep when merged: https://github.com/Textualize/toolong/pull/47 +# toolong +git+https://github.com/ewels/toolong.git@add-entry-points#egg=toolong From 7f814b621e505ff60077a24d58f0ca4353dd7c11 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 24 Mar 2024 00:26:11 +0100 Subject: [PATCH 07/12] Fix github syntax for requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25df1a95d3..081847d655 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ tabulate trogon # Remove dev brach dep when merged: https://github.com/Textualize/toolong/pull/47 # toolong -git+https://github.com/ewels/toolong.git@add-entry-points#egg=toolong +toolong @ git+https://github.com/ewels/toolong.git@add-entry-points#egg=toolong From 5e4b54d4d924af4a0515d76e03792c8ab51c3096 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 25 Mar 2024 21:36:17 +0100 Subject: [PATCH 08/12] Add docstrings with examples for each regex --- nf_core/toolong_formatter.py | 123 ++++++++++++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 8 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index ab35a1d4eb..a12724a5ea 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -36,6 +36,15 @@ def parse(self, line: str) -> ParseResult | None: class NextflowLogFormat(LogFormat): + """ + Formatter for regular Nextflow log files. + + Examples: + + Mar-24 00:11:47.302 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 11; maxThreads: 1000 + Mar-24 00:12:04.942 [Task monitor] INFO nextflow.Session - Execution cancelled -- Finishing pending tasks before exit + """ + REGEX = re.compile( r"(?P\w+-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) (?P\[.*\]?) (?P\w+)\s+(?P[\w\.]+) - (?P.*?)$" ) @@ -66,7 +75,36 @@ def parse(self, line: str) -> ParseResult | None: return timestamp, line, text -class NextflowLogFormatActiveProcess(LogFormat): +class NextflowLogAbortedProcessNames(LogFormat): + """ + Formatter for process names when a pipeline is aborted. + + Example: + + The following lines: + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT + + In blocks that look like this: + + Mar-12 23:56:10.538 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT + Mar-12 23:56:10.572 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (value) bound ; channel: - + port 2: (value) bound ; channel: - + port 3: (value) bound ; channel: star_ignore_sjdbgtf + port 4: (value) bound ; channel: seq_platform + port 5: (value) bound ; channel: seq_center + port 6: (cntrl) - ; channel: $ + + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (cntrl) - ; channel: $ + """ + REGEX = re.compile(r"^(?P\[process\]) (?P.*?)(?P[^:]+?)?$") highlighter = LogHighlighter() @@ -86,7 +124,38 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowLogFormatActiveProcessDetails(LogFormat): +class NextflowLogAbortedProcessPorts(LogFormat): + """ + Formatter for process names when a pipeline is aborted. + + Example: + + The following lines: + port 0: (queue) OPEN ; channel: - + port 1: (value) bound ; channel: - + port 2: (value) bound ; channel: - + port 3: (value) bound ; channel: star_ignore_sjdbgtf + + In blocks that look like this: + + Mar-12 23:56:10.538 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT + Mar-12 23:56:10.572 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (value) bound ; channel: - + port 2: (value) bound ; channel: - + port 3: (value) bound ; channel: star_ignore_sjdbgtf + port 4: (value) bound ; channel: seq_platform + port 5: (value) bound ; channel: seq_center + port 6: (cntrl) - ; channel: $ + + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (cntrl) - ; channel: $ + """ + REGEX = re.compile( r" (?Pport \d+): (?P\((value|queue|cntrl)\)) (?P\S+)\s+; channel: (?P.*?)$" ) @@ -117,7 +186,35 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowLogFormatActiveProcessStatus(LogFormat): +class NextflowLogAbortedProcessStatus(LogFormat): + """ + Formatter for process names when a pipeline is aborted. + + Example: + + The following lines: + status=ACTIVE + + In blocks that look like this: + + Mar-12 23:56:10.538 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT + Mar-12 23:56:10.572 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (value) bound ; channel: - + port 2: (value) bound ; channel: - + port 3: (value) bound ; channel: star_ignore_sjdbgtf + port 4: (value) bound ; channel: seq_platform + port 5: (value) bound ; channel: seq_center + port 6: (cntrl) - ; channel: $ + + [process] NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT + status=ACTIVE + port 0: (queue) OPEN ; channel: - + port 1: (cntrl) - ; channel: $ + """ + REGEX = re.compile(r"^ status=(?P.*?)?$") highlighter = LogHighlighter() @@ -136,7 +233,17 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -class NextflowLogFormatScriptParse(LogFormat): +class NextflowLogParsedScripts(LogFormat): + """ + Formatter for parsed scriptp names. + + For example: + + Mar-24 00:12:03.547 [main] DEBUG nextflow.script.ScriptRunner - Parsed script files: + Script_e2630658c898fe40: /Users/ewels/GitHub/nf-core/rnaseq/./workflows/rnaseq/../../modules/local/deseq2_qc/main.nf + Script_56c7c9e8363ee20a: /Users/ewels/GitHub/nf-core/rnaseq/./workflows/rnaseq/../../subworkflows/local/quantify_pseudo_alignment/../../../modules/nf-core/custom/tx2gene/main.nf + """ + REGEX = re.compile(r"^ (?PScript_\w+:) (?P.*?)$") highlighter = LogHighlighter() @@ -162,10 +269,10 @@ def nextflow_formatters(formats): if nf_core.toolong_formatter.IS_NEXTFLOW: return [ NextflowLogFormat(), - NextflowLogFormatActiveProcess(), - NextflowLogFormatActiveProcessDetails(), - NextflowLogFormatActiveProcessStatus(), - NextflowLogFormatScriptParse(), + NextflowLogAbortedProcessNames(), + NextflowLogAbortedProcessPorts(), + NextflowLogAbortedProcessStatus(), + NextflowLogParsedScripts(), ] return formats From 2079df9d3bdbfd51c4158bf21865ecdc7a551bb5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 25 Mar 2024 23:38:00 +0100 Subject: [PATCH 09/12] Refactor to use one entry point, better nextflow log filename detection --- nf_core/toolong_formatter.py | 43 ++++++++++++------------------------ setup.py | 6 ----- 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index a12724a5ea..f298028ebb 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -5,6 +5,7 @@ from typing import Optional from rich.text import Text +from toolong.format_parser import FormatParser from toolong.highlighter import LogHighlighter from typing_extensions import TypeAlias @@ -19,16 +20,6 @@ "ERROR": ["bold black on red", "red"], } -IS_NEXTFLOW = False - - -def nf_toolong_on_init(scan): - import nf_core.toolong_formatter - - for file_path in scan.file_paths: - if file_path.startswith(".nextflow.log"): - nf_core.toolong_formatter.IS_NEXTFLOW = True - class LogFormat: def parse(self, line: str) -> ParseResult | None: @@ -263,28 +254,22 @@ def parse(self, line: str) -> ParseResult | None: return None, line, text -def nextflow_formatters(formats): - import nf_core.toolong_formatter - - if nf_core.toolong_formatter.IS_NEXTFLOW: - return [ - NextflowLogFormat(), - NextflowLogAbortedProcessNames(), - NextflowLogAbortedProcessPorts(), - NextflowLogAbortedProcessStatus(), - NextflowLogParsedScripts(), - ] - return formats - - -def nextflow_format_parser(format_parser): - import nf_core.toolong_formatter +def nextflow_format_parser(logfile_obj): + is_nextflow = logfile_obj.name.startswith(".nextflow.log") - class FormatParser(format_parser): + class NextflowFormatParser(FormatParser): """Parses a log line.""" def __init__(self) -> None: super().__init__() + if is_nextflow: + self._formats = [ + NextflowLogFormat(), + NextflowLogAbortedProcessNames(), + NextflowLogAbortedProcessPorts(), + NextflowLogAbortedProcessStatus(), + NextflowLogParsedScripts(), + ] self._log_status = "" def parse(self, line: str) -> ParseResult: @@ -294,7 +279,7 @@ def parse(self, line: str) -> ParseResult: timestamp, line, text = super().parse(line) # Return if not a netflow log file - if not nf_core.toolong_formatter.IS_NEXTFLOW: + if not is_nextflow: return timestamp, line, text # Custom formatting with log levels @@ -327,4 +312,4 @@ def parse(self, line: str) -> ParseResult: return timestamp, line, text - return FormatParser + return NextflowFormatParser() diff --git a/setup.py b/setup.py index b7ed6ea1f2..d9fde6d9fd 100644 --- a/setup.py +++ b/setup.py @@ -34,15 +34,9 @@ entry_points={ "console_scripts": ["nf-core=nf_core.__main__:run_nf_core"], "refgenie.hooks.post_update": ["nf-core-refgenie=nf_core.refgenie:update_config"], - "toolong.application.formats": [ - "nextflow_formatters = nf_core.toolong_formatter:nextflow_formatters", - ], "toolong.application.format_parsers": [ "nextflow_format_parser = nf_core.toolong_formatter:nextflow_format_parser", ], - "toolong.application.on_init": [ - "nf_toolong_on_init = nf_core.toolong_formatter:nf_toolong_on_init", - ], }, python_requires=">=3.8, <4", install_requires=required, From f0b27686580db51cc23d6eb5e3c912a0e16950e3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 25 Mar 2024 23:47:54 +0100 Subject: [PATCH 10/12] Fix secondary formatter types --- nf_core/toolong_formatter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index f298028ebb..8572fe59a4 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -11,7 +11,6 @@ ParseResult: TypeAlias = "tuple[Optional[datetime], str, Text]" - MOVE_LOG_LEVEL_COL = True LOG_LEVELS = { "DEBUG": ["dim white on black", "dim"], @@ -244,7 +243,6 @@ def parse(self, line: str) -> ParseResult | None: return None text = Text.from_ansi(line) - text.stylize_before("dim") groups = match.groupdict() if script_id := groups.get("script_id", None): text.highlight_words([script_id], "blue") @@ -304,7 +302,6 @@ def parse(self, line: str) -> ParseResult: # Multi-line log message # Strip automatic formatting, which does weird stuff - text = Text(line) for logtype in LOG_LEVELS.keys(): if self._log_status == logtype: text = Text.from_markup(f"[{LOG_LEVELS[logtype][0]}] [/] " + text.markup) From afc771c61725a6d64d3a1ece23e8fa3de2ba750e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Mar 2024 00:01:31 +0100 Subject: [PATCH 11/12] Remove automatic tl formatting --- nf_core/toolong_formatter.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/nf_core/toolong_formatter.py b/nf_core/toolong_formatter.py index 8572fe59a4..cca2fce27a 100644 --- a/nf_core/toolong_formatter.py +++ b/nf_core/toolong_formatter.py @@ -271,14 +271,29 @@ def __init__(self) -> None: self._log_status = "" def parse(self, line: str) -> ParseResult: - """Parse a line.""" - - # Use the toolong parser with custom formatters - timestamp, line, text = super().parse(line) + """Use the toolong parser with custom formatters.""" # Return if not a netflow log file if not is_nextflow: - return timestamp, line, text + return super().parse(line) + + # Copied from toolong source, but without the default log parser + if len(line) > 10_000: + line = line[:10_000] + parse_result = None + if line.strip(): + for index, format in enumerate(self._formats): + parse_result = format.parse(line) + if parse_result is not None: + if index: + self._formats = [*self._formats[index:], *self._formats[:index]] + timestamp, line, text = parse_result + break + + if parse_result is None: + timestamp = None + line = line + text = Text(line) # Custom formatting with log levels for logtype in LOG_LEVELS.keys(): @@ -300,8 +315,7 @@ def parse(self, line: str) -> ParseResult: # Return - on to next line return timestamp, line, text - # Multi-line log message - # Strip automatic formatting, which does weird stuff + # Multi-line log message - add colour character at start of line for logtype in LOG_LEVELS.keys(): if self._log_status == logtype: text = Text.from_markup(f"[{LOG_LEVELS[logtype][0]}] [/] " + text.markup) From 74820e666ac37dd5a1b515fcf1c73d90050a853c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Mar 2024 00:32:31 +0100 Subject: [PATCH 12/12] Add new 'nf-core log' command to launch Toolong --- nf_core/__main__.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 807bc776bb..42398bfcb1 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -3,6 +3,7 @@ import logging import os +import subprocess import sys from pathlib import Path @@ -36,6 +37,7 @@ "commands": [ "list", "launch", + "log", "create-params-file", "download", "licences", @@ -311,6 +313,34 @@ def launch( sys.exit(1) +# nf-core log +@nf_core_cli.command("log") +@click.argument("filenames", required=False, nargs=-1, metavar="") +def view_logs(filenames): + """ + Render .nextflow.log files nicely. + + Uses [link=https://github.com/textualize/toolong/]Toolong[/] with the included nf-core extension. + Shows files globbed with `.nextflow.log*` by default, or supplied filenames. + """ + filenames = list(filenames) + if len(filenames) == 0: + p = Path(".") + p.glob(".nextflow.log*") + filenames = [str(f) for f in p.glob(".nextflow.log*")] + + if len(filenames) == 0: + log.error("No .nextflow.log files found.") + sys.exit(1) + + filenames.sort() + log.info(f"Launching log viewer for: [green]{", ".join(filenames)}") + log.info( + "This uses the [link=https://github.com/textualize/toolong/]Toolong[/] log viewer - you can view any file with it using the `[magenta]tl[/]` command!" + ) + subprocess.run(["tl", *filenames]) + + # nf-core create-params-file @nf_core_cli.command() @click.argument("pipeline", required=False, metavar="")