From afdbfa53de770023708bafed5e2ce43616ef0b1c Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Thu, 16 Apr 2026 16:18:58 +0200 Subject: [PATCH 1/9] preparing rule detector --- .../detectors/rule_detector.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/detectmatelibrary/detectors/rule_detector.py diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py new file mode 100644 index 0000000..c620c00 --- /dev/null +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -0,0 +1,31 @@ +from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig + +from detectmatelibrary.utils.data_buffer import BufferMode + +from detectmatelibrary import schemas + +from typing import List, Any + + +class RuleDetectorConfig(CoreDetectorConfig): + method_type: str = "dummy_detector" + + +class RuleDetector(CoreDetector): + def __init__( + self, + name: str = "RuleDetector", + config: RuleDetectorConfig | dict[str, Any] = RuleDetectorConfig() + ) -> None: + + if isinstance(config, dict): + config = RuleDetectorConfig.from_dict(config, name) + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + self._call_count = 0 + + def detect( + self, + input_: List[schemas.ParserSchema] | schemas.ParserSchema, + output_: schemas.DetectorSchema + ) -> bool: + return False From d1a1021ae547c4d9fdfdc72eeb6e90646bb0bd53 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Thu, 16 Apr 2026 17:11:47 +0200 Subject: [PATCH 2/9] start adding rules --- .../detectors/rule_detector.py | 35 ++++++++++++++++++- tests/test_detectors/test_rule_detector.py | 32 +++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/test_detectors/test_rule_detector.py diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index c620c00..356e063 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -1,3 +1,9 @@ +"""Copy and paste to create new rules. + +def template_rule(input_: schemas.ParserSchema, *args: list[Any]) -> +tuple[bool, str]: raise_alert = False # Add here rule message = +"" # Rule message return raise_alert, message +""" from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig from detectmatelibrary.utils.data_buffer import BufferMode @@ -7,8 +13,35 @@ from typing import List, Any +def template_not_found(input_: schemas.ParserSchema, *args: list[Any]) -> tuple[bool, str]: + raise_alert = input_["EventID"] == -1 + message = "Template was not found by the parser" + return raise_alert, message + + +def find_keyword(input_: schemas.ParserSchema, args: list[str]) -> tuple[bool, str]: + log: str = input_["log"] + log = log.lower() + + for k in args: + if k in log: + return True, f"Found word '{k}' in the logs" + return False, "" + + +def exceptions(input_: schemas.ParserSchema, *args: list[Any]) -> tuple[bool, str]: + return find_keyword(input_, ["exception", "fail", "error", "raise"]) + + +rules = { + "TemplateNotFound": template_not_found, + "SpecificKeyword": find_keyword, + "CheckForExceptions": exceptions, +} + + class RuleDetectorConfig(CoreDetectorConfig): - method_type: str = "dummy_detector" + method_type: str = "rule_detector" class RuleDetector(CoreDetector): diff --git a/tests/test_detectors/test_rule_detector.py b/tests/test_detectors/test_rule_detector.py new file mode 100644 index 0000000..59a7b5a --- /dev/null +++ b/tests/test_detectors/test_rule_detector.py @@ -0,0 +1,32 @@ +import detectmatelibrary.detectors.rule_detector as rd +import detectmatelibrary.schemas as schemas + + +class TestCaseRules: + def test_template_not_found(self) -> None: + parsed_log = schemas.ParserSchema({"EventID": -1}) + + alert, msg = rd.template_not_found(parsed_log) + assert alert + assert "Template was not found by the parser" == msg + + parsed_log = schemas.ParserSchema({"EventID": 2}) + alert, _ = rd.template_not_found(parsed_log) + assert not alert + + def test_find_keyword(self) -> None: + keywords = ["hello", "ciao"] + + parsed_log = schemas.ParserSchema({"log": "hello world"}) + alert, msg = rd.find_keyword(parsed_log, *keywords) + assert alert + assert "Found word 'hello' in the logs" == msg + + parsed_log = schemas.ParserSchema({"log": "ciao world"}) + alert, msg = rd.find_keyword(parsed_log, *keywords) + assert alert + assert "Found word 'ciao' in the logs" == msg + + parsed_log = schemas.ParserSchema({"log": "world"}) + alert, _ = rd.find_keyword(parsed_log, *keywords) + assert not alert From 51d4e4a90cf83998270447ad554452891d5e51f8 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Thu, 16 Apr 2026 17:13:53 +0200 Subject: [PATCH 3/9] minor bug issue generated by MYPY! --- src/detectmatelibrary/detectors/rule_detector.py | 1 - tests/test_detectors/test_rule_detector.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index 356e063..fe1737b 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -54,7 +54,6 @@ def __init__( if isinstance(config, dict): config = RuleDetectorConfig.from_dict(config, name) super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) - self._call_count = 0 def detect( self, diff --git a/tests/test_detectors/test_rule_detector.py b/tests/test_detectors/test_rule_detector.py index 59a7b5a..056c1ed 100644 --- a/tests/test_detectors/test_rule_detector.py +++ b/tests/test_detectors/test_rule_detector.py @@ -18,15 +18,15 @@ def test_find_keyword(self) -> None: keywords = ["hello", "ciao"] parsed_log = schemas.ParserSchema({"log": "hello world"}) - alert, msg = rd.find_keyword(parsed_log, *keywords) + alert, msg = rd.find_keyword(parsed_log, keywords) assert alert assert "Found word 'hello' in the logs" == msg parsed_log = schemas.ParserSchema({"log": "ciao world"}) - alert, msg = rd.find_keyword(parsed_log, *keywords) + alert, msg = rd.find_keyword(parsed_log, keywords) assert alert assert "Found word 'ciao' in the logs" == msg parsed_log = schemas.ParserSchema({"log": "world"}) - alert, _ = rd.find_keyword(parsed_log, *keywords) + alert, _ = rd.find_keyword(parsed_log, keywords) assert not alert From 07061e9e02992a77a81fa686c57c6033d45b7fb6 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Thu, 16 Apr 2026 17:15:29 +0200 Subject: [PATCH 4/9] improve small formatting issue --- src/detectmatelibrary/detectors/rule_detector.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index fe1737b..cee5945 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -1,8 +1,12 @@ """Copy and paste to create new rules. -def template_rule(input_: schemas.ParserSchema, *args: list[Any]) -> -tuple[bool, str]: raise_alert = False # Add here rule message = -"" # Rule message return raise_alert, message +def template_rule( + input_: schemas.ParserSchema, *args: list[Any] +) -> tuple[bool, str]: + + raise_alert = False # Add here rule + message = "" # Rule message + return raise_alert, message """ from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig From a6ec3373e6fdf99b68944c718c9ef966f9cbe016 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Fri, 17 Apr 2026 10:09:28 +0200 Subject: [PATCH 5/9] add more rules --- .../detectors/rule_detector.py | 20 +++++++++++-- tests/test_detectors/test_rule_detector.py | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index cee5945..71184df 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -37,15 +37,29 @@ def exceptions(input_: schemas.ParserSchema, *args: list[Any]) -> tuple[bool, st return find_keyword(input_, ["exception", "fail", "error", "raise"]) +def error_log(input_: schemas.ParserSchema, *args: list[Any]) -> tuple[bool, str]: + vars_: dict[str, str] = input_["logFormatVariables"] + raise_alert = "Level" in vars_ and "error" == vars_["Level"].lower() + message = "Error found" + return raise_alert, message + + rules = { - "TemplateNotFound": template_not_found, - "SpecificKeyword": find_keyword, - "CheckForExceptions": exceptions, + "R001 - TemplateNotFound": template_not_found, + "R002 - SpecificKeyword": find_keyword, + "R003 - CheckForExceptions": exceptions, + "R004 - ErrorLevelFound": error_log, } class RuleDetectorConfig(CoreDetectorConfig): method_type: str = "rule_detector" + rules: list[dict[str, list[str] | str]] = [ + {"rule": "R001 - TemplateNotFound"}, + {"rule": "R002 - SpecificKeyword", "args": ["searching for wally"]}, + {"rule": "R003 - CheckForExceptions"}, + {"rule": "R004 - ErrorLevelFound"}, + ] class RuleDetector(CoreDetector): diff --git a/tests/test_detectors/test_rule_detector.py b/tests/test_detectors/test_rule_detector.py index 056c1ed..555fc5f 100644 --- a/tests/test_detectors/test_rule_detector.py +++ b/tests/test_detectors/test_rule_detector.py @@ -30,3 +30,31 @@ def test_find_keyword(self) -> None: parsed_log = schemas.ParserSchema({"log": "world"}) alert, _ = rd.find_keyword(parsed_log, keywords) assert not alert + + def test_find_exceptions(self) -> None: + parsed_log = schemas.ParserSchema({"log": "hello Exception"}) + alert, msg = rd.exceptions(parsed_log) + assert alert + assert "Found word 'exception' in the logs" == msg + + parsed_log = schemas.ParserSchema({"log": "world"}) + alert, _ = rd.exceptions(parsed_log) + assert not alert + + def test_error_log(self) -> None: + parsed_log = schemas.ParserSchema( + {"logFormatVariables": {"Level": "Error"}} + ) + alert, msg = rd.error_log(parsed_log) + assert alert + assert "Error found" == msg + + parsed_log = schemas.ParserSchema( + {"logFormatVariables": {"Level": "Info"}} + ) + alert, _ = rd.error_log(parsed_log) + assert not alert + + parsed_log = schemas.ParserSchema({"log": "hello world"}) + alert, _ = rd.error_log(parsed_log) + assert not alert From 9669d66c507340b18fa7dd3f9d910f60b4c8ed0c Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 21 Apr 2026 12:33:11 +0200 Subject: [PATCH 6/9] rule detector ready --- .../detectors/rule_detector.py | 30 ++++- tests/test_detectors/test_rule_detector.py | 120 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index 71184df..21e90ff 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -27,6 +27,7 @@ def find_keyword(input_: schemas.ParserSchema, args: list[str]) -> tuple[bool, s log: str = input_["log"] log = log.lower() + print(args) for k in args: if k in log: return True, f"Found word '{k}' in the logs" @@ -52,11 +53,15 @@ def error_log(input_: schemas.ParserSchema, *args: list[Any]) -> tuple[bool, str } +class RuleNotFound(Exception): + def __init__(self, rule: str) -> None: + super().__init__(f"Rule -> ([{rule}]) not found") + + class RuleDetectorConfig(CoreDetectorConfig): method_type: str = "rule_detector" rules: list[dict[str, list[str] | str]] = [ {"rule": "R001 - TemplateNotFound"}, - {"rule": "R002 - SpecificKeyword", "args": ["searching for wally"]}, {"rule": "R003 - CheckForExceptions"}, {"rule": "R004 - ErrorLevelFound"}, ] @@ -72,10 +77,31 @@ def __init__( if isinstance(config, dict): config = RuleDetectorConfig.from_dict(config, name) super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + self.config: RuleDetectorConfig + + for rule in self.config.rules: + if rule["rule"] not in rules: + raise RuleNotFound(rule) def detect( self, input_: List[schemas.ParserSchema] | schemas.ParserSchema, output_: schemas.DetectorSchema ) -> bool: - return False + + anomaly = False + output_["score"] = 0 + + for rule in self.config.rules: + if "args" in rule: + alert, msg = rules[rule["rule"]](input_, rule["args"]) # type: ignore + else: + alert, msg = rules[rule["rule"]](input_) # type: ignore + + if alert: + output_["alertsObtain"][rule["rule"]] = msg + output_["score"] += 1 + + anomaly = anomaly or alert + + return anomaly diff --git a/tests/test_detectors/test_rule_detector.py b/tests/test_detectors/test_rule_detector.py index 555fc5f..43a1b77 100644 --- a/tests/test_detectors/test_rule_detector.py +++ b/tests/test_detectors/test_rule_detector.py @@ -1,6 +1,8 @@ import detectmatelibrary.detectors.rule_detector as rd import detectmatelibrary.schemas as schemas +import pytest + class TestCaseRules: def test_template_not_found(self) -> None: @@ -58,3 +60,121 @@ def test_error_log(self) -> None: parsed_log = schemas.ParserSchema({"log": "hello world"}) alert, _ = rd.error_log(parsed_log) assert not alert + + +class TestCaseRuleDetector: + def test_run_all_rules(self) -> None: + rule_detector = rd.RuleDetector() + + assert rule_detector.process(schemas.ParserSchema()) is None + + alert1 = rule_detector.process(schemas.ParserSchema( + { + "EventID": -1, + "log": "fail exception", + "logFormatVariables": {"Level": "Error"} + } + )) + assert alert1 is not None + assert alert1["alertsObtain"] == { + "R001 - TemplateNotFound": "Template was not found by the parser", + "R003 - CheckForExceptions": "Found word 'exception' in the logs", + "R004 - ErrorLevelFound": "Error found", + } + assert alert1["score"] == 3 + + def test_all_rules_serialize(self) -> None: + rule_detector = rd.RuleDetector() + + assert rule_detector.process(schemas.ParserSchema().serialize()) is None + + alert1 = rule_detector.process(schemas.ParserSchema( + { + "EventID": -1, + "log": "fail exception", + "logFormatVariables": {"Level": "Error"} + } + ).serialize()) + assert alert1 is not None + + (alert := schemas.DetectorSchema()).deserialize(alert1) + assert alert["alertsObtain"] == { + "R001 - TemplateNotFound": "Template was not found by the parser", + "R003 - CheckForExceptions": "Found word 'exception' in the logs", + "R004 - ErrorLevelFound": "Error found", + } + assert alert["score"] == 3 + + def test_run_single_rule(self) -> None: + rule_detector = rd.RuleDetector( + name="RuleDetector", + config={ + "detectors": { + "RuleDetector": { + "method_type": "rule_detector", + "auto_config": False, + "params": { + "rules": [ + {"rule": "R001 - TemplateNotFound"} + ] + } + } + } + } + ) + + assert rule_detector.process(schemas.ParserSchema()) is None + + alert1 = rule_detector.process(schemas.ParserSchema( + { + "EventID": -1, + "log": "fail exception", + "logFormatVariables": {"Level": "Error"} + } + )) + assert alert1 is not None + assert alert1["alertsObtain"] == { + "R001 - TemplateNotFound": "Template was not found by the parser", + } + assert alert1["score"] == 1 + + def test_run_single_rule_with_args(self) -> None: + rule_detector = rd.RuleDetector( + name="RuleDetector", + config={ + "detectors": { + "RuleDetector": { + "method_type": "rule_detector", + "auto_config": False, + "params": { + "rules": [ + {"rule": "R002 - SpecificKeyword", "args": ["hi", "kenobi"]} + ] + } + } + } + } + ) + + assert rule_detector.process(schemas.ParserSchema({"log": "ciao"})) is None + assert rule_detector.process(schemas.ParserSchema({"log": "hi"})) is not None + + def test_rule_not_found(self) -> None: + + with pytest.raises(rd.RuleNotFound): + rd.RuleDetector( + name="RuleDetector", + config={ + "detectors": { + "RuleDetector": { + "method_type": "rule_detector", + "auto_config": False, + "params": { + "rules": [ + {"rule": "Rule made up"} + ] + } + } + } + } + ) From de1401bbdb07ce6f4d8ee4f754121489d7429db4 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 21 Apr 2026 12:33:57 +0200 Subject: [PATCH 7/9] remove print --- src/detectmatelibrary/detectors/rule_detector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/detectmatelibrary/detectors/rule_detector.py b/src/detectmatelibrary/detectors/rule_detector.py index 21e90ff..1143eb2 100644 --- a/src/detectmatelibrary/detectors/rule_detector.py +++ b/src/detectmatelibrary/detectors/rule_detector.py @@ -27,7 +27,6 @@ def find_keyword(input_: schemas.ParserSchema, args: list[str]) -> tuple[bool, s log: str = input_["log"] log = log.lower() - print(args) for k in args: if k in log: return True, f"Found word '{k}' in the logs" From 618ddb0aeb9fb87cd83c0a263807258fae50da17 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 21 Apr 2026 12:42:47 +0200 Subject: [PATCH 8/9] prepare documentation --- docs/detectors.md | 1 + docs/detectors/rule_based.md | 0 mkdocs.yml | 2 ++ 3 files changed, 3 insertions(+) create mode 100644 docs/detectors/rule_based.md diff --git a/docs/detectors.md b/docs/detectors.md index 7625b9d..2a3231c 100644 --- a/docs/detectors.md +++ b/docs/detectors.md @@ -88,6 +88,7 @@ List of detectors: * [New Value](detectors/new_value.md): Detect new values in the variables in the logs. * [Combo Detector](detectors/combo.md): Detect new combination of variables in the logs. * [New Event](detectors/new_event.md): Detect new events in the variables in the logs. +* [Rule Based](detectors/rule_based.md): Detect anomalies based in a set of rules. ## Configuration diff --git a/docs/detectors/rule_based.md b/docs/detectors/rule_based.md new file mode 100644 index 0000000..e69de29 diff --git a/mkdocs.yml b/mkdocs.yml index e8af2af..ba9e1f8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,6 +24,8 @@ nav: - Random Detector: detectors/random_detector.md - New Value: detectors/new_value.md - Combo Detector: detectors/combo.md + - New Event: detectors/new_event.md + - Rule Based: detectors/rule_based.md - Auxiliar: - Persistency: auxiliar/persistency.md - Data buffer: auxiliar/input_buffer.md From 53d3b79d84ac2739609832d3517a3355fbde4a1e Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 21 Apr 2026 13:20:26 +0200 Subject: [PATCH 9/9] add rule based method documentation --- docs/detectors/rule_based.md | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/detectors/rule_based.md b/docs/detectors/rule_based.md index e69de29..5c33f9d 100644 --- a/docs/detectors/rule_based.md +++ b/docs/detectors/rule_based.md @@ -0,0 +1,69 @@ +# Rule-based Detector + +The Rule-based Detector raises alerts based on a configurable set of rules. + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [ParserSchema](../schemas.md) | Structured log | +| **Output** | [DetectorSchema](../schemas.md) | Alert / finding | + +## Description + +The detector analyzes parsed logs one by one and checks which rules are triggered. When alerts are produced, the triggered rules and their messages are recorded in the `alertsObtain` field of the output schema. The `score` field contains the number of rules that triggered. + +### Available rules + +| Rule name | Description | Requires arguments | Enabled by default | +|---|---|---:|:---:| +| **R001 - TemplateNotFound** | Check whether the parser assigned a template to the log | No | Yes | +| **R002 - SpecificKeyword** | Check for one or more user-specified keywords in the log content | list of words | No | +| **R003 - CheckForExceptions** | Check for words commonly associated with exceptions or failures | No | Yes | +| **R004 - ErrorLevelFound** | If a Level field exists, check whether it indicates an error level | No | No | + +Notes on table columns: + +- **Rule name**: Identifier used in configuration. +- **Description**: What the rule checks. +- **Requires arguments**: Whether the rule needs additional arguments. +- **Enabled by default**: Whether the rule is active when not explicitly overridden. + +## Configuration example + +```yaml +detectors: + RuleDetector: + method_type: rule_detector + auto_config: False + params: + rules: + - rule: "R001 - TemplateNotFound" + - rule: "R002 - SpecificKeyword" + args: + - "critical" + - "anomaly" +``` + +## Example usage + +```python +import detectmatelibrary.detectors.rule_detector as rd +from detectmatelibrary import schemas + +rule_detector = rd.RuleDetector() + +parser_data = schemas.ParserSchema({ + "parserType": "test", + "EventID": 1, + "template": "test template", + "variables": ["var1"], + "logID": "1", + "parsedLogID": "1", + "parserID": "test_parser", + "log": "test log message", + "logFormatVariables": {"timestamp": "123456"} +}) + +alert = rule_detector.process(parser_data) +``` + +Go back [Index](../index.md)