From 1d3b7ed0d3d863dd55d49e836dfc7f314d9ab26b Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 12:15:14 -0400 Subject: [PATCH 1/6] Add fine grain 'keep' req bypass --- CLI.md | 2 ++ detection_rules/rule.py | 47 +++++++++++++++----------- detection_rules/utils.py | 1 + docs-dev/custom-rules-management.md | 2 +- docs-dev/developing.md | 2 ++ pyproject.toml | 2 +- tests/test_schemas.py | 51 +++++++++++++++++++++++++++-- 7 files changed, 83 insertions(+), 24 deletions(-) diff --git a/CLI.md b/CLI.md index dd9df97179b..e331b86272d 100644 --- a/CLI.md +++ b/CLI.md @@ -46,6 +46,8 @@ Using the environment variable `DR_BYPASS_TAGS_VALIDATION` will bypass the Detec Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will bypass the timeline template id and title validation for rules. +Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass local validation that ES|QL rules include a `keep` command and that non-aggregate queries list `_id`, `_version`, and `_index` in `keep` (other ES|QL checks are unchanged). + Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI. For instance, some users may want to increase the default value in cases where help messages are cut off. diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 091e1f11a67..11a3e5dd76f 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -989,28 +989,35 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None: ) # Enforce KEEP command for ESQL rules and that METADATA fields are present in non-aggregate queries - # Match | followed by optional whitespace/newlines and then 'keep' - keep_pattern = re.compile(r"\|\s*keep\b\s+([^\|]+)", re.IGNORECASE | re.DOTALL) - keep_matches = list(keep_pattern.finditer(query_lower)) - if not keep_matches: - raise EsqlSemanticError( - f"Rule: {data['name']} does not contain a 'keep' command -> Add a 'keep' command to the query." + if os.environ.get("DR_BYPASS_ESQL_KEEP_VALIDATION") is None: + bypass_keep_hint = ( + " To bypass ES|QL `keep` validation, set the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION`." ) + # Match | followed by optional whitespace/newlines and then 'keep' + keep_pattern = re.compile(r"\|\s*keep\b\s+([^\|]+)", re.IGNORECASE | re.DOTALL) + keep_matches = list(keep_pattern.finditer(query_lower)) + if not keep_matches: + raise EsqlSemanticError( + f"Rule: {data['name']} does not contain a 'keep' command -> Add a 'keep' command to the query." + + bypass_keep_hint + ) - # Ensure that keep clause includes metadata fields on non-aggregate queries - aggregate_pattern = re.compile(r"\|\s*stats\b(?:\s+([^\|]+?))?(?:\s+by\s+([^\|]+))?", re.IGNORECASE | re.DOTALL) - if not aggregate_pattern.search(query_lower): - for keep_match in keep_matches: - raw_keep = re.sub(r"//.*", "", keep_match.group(1)) - keep_fields = [field.strip() for field in raw_keep.split(",") if field.strip()] - if "*" not in keep_fields: - required_metadata = {"_id", "_version", "_index"} - if not required_metadata.issubset(set(map(str.strip, keep_fields))): - raise EsqlSemanticError( - f"Rule: {data['name']} contains a keep clause without" - f" metadata fields '_id', '_version', and '_index' ->" - f" Add '_id', '_version', '_index' to the keep command." - ) + # Ensure that keep clause includes metadata fields on non-aggregate queries + aggregate_pattern = re.compile( + r"\|\s*stats\b(?:\s+([^\|]+?))?(?:\s+by\s+([^\|]+))?", re.IGNORECASE | re.DOTALL + ) + if not aggregate_pattern.search(query_lower): + for keep_match in keep_matches: + raw_keep = re.sub(r"//.*", "", keep_match.group(1)) + keep_fields = [field.strip() for field in raw_keep.split(",") if field.strip()] + if "*" not in keep_fields: + required_metadata = {"_id", "_version", "_index"} + if not required_metadata.issubset(set(map(str.strip, keep_fields))): + raise EsqlSemanticError( + f"Rule: {data['name']} contains a keep clause without" + f" metadata fields '_id', '_version', and '_index' ->" + f" Add '_id', '_version', '_index' to the keep command." + bypass_keep_hint + ) @dataclass(frozen=True, kw_only=True) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index eedd45d3191..916db9434c0 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -142,6 +142,7 @@ def set_all_validation_bypass(env_value: bool = False) -> None: os.environ["DR_BYPASS_BBR_LOOKBACK_VALIDATION"] = str(env_value) os.environ["DR_BYPASS_TAGS_VALIDATION"] = str(env_value) os.environ["DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION"] = str(env_value) + os.environ["DR_BYPASS_ESQL_KEEP_VALIDATION"] = str(env_value) def set_nested_value(obj: dict[str, Any], compound_key: str, value: Any) -> None: diff --git a/docs-dev/custom-rules-management.md b/docs-dev/custom-rules-management.md index c95a103f361..a9222e4c189 100644 --- a/docs-dev/custom-rules-management.md +++ b/docs-dev/custom-rules-management.md @@ -76,7 +76,7 @@ Some notes: * To manage action-connectors tied to rules one can set an action-connectors directory using the optional `action_connector_dir` value (included above) set to be the desired path. If an actions_connector directory is explicitly specified in a CLI command, the config value will be ignored. * To turn on automatic schema generation for non-ecs fields via custom schemas add `auto_gen_schema_file: `. This will generate a schema file in the specified location that will be used to add entries for each field and index combination that is not already in a known schema. This will also automatically add it to your stack-schema-map.yaml file when using a custom rules directory and config. * For Kibana action items, currently these are included in the rule toml files themselves. At a later date, we may allow for bulk editing of rule action items through separate action toml files. The action_dir config key is left available for this later implementation. For now to bulk update, use the bulk actions add rule actions UI in Kibana. -* To on bulk disable elastic validation for optional fields, use the following line `bypass_optional_elastic_validation: True`. +* To on bulk disable elastic validation for optional fields, use the following line `bypass_optional_elastic_validation: True`. That sets the shared `DR_BYPASS_*` toggles (including note, BBR lookback, tags, timeline template, and ES|QL `keep` validation). When using the repo, set the environment variable `CUSTOM_RULES_DIR=` diff --git a/docs-dev/developing.md b/docs-dev/developing.md index 815ad37f155..3cf3ef5bff5 100644 --- a/docs-dev/developing.md +++ b/docs-dev/developing.md @@ -48,6 +48,8 @@ Using the environment variable `DR_BYPASS_TAGS_VALIDATION` will bypass the Detec Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will bypass the timeline template id and title validation for rules. +Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass local validation that ES|QL rules include a `keep` command and that non-aggregate queries list `_id`, `_version`, and `_index` in `keep` (other ES|QL checks are unchanged). + ## Using the `RuleResource` methods built on detections `_bulk_action` APIs diff --git a/pyproject.toml b/pyproject.toml index 02b3bbd6268..6e030095472 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.6" +version = "1.6.7" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 5f5e609eb94..b365a0dcaa4 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -6,13 +6,14 @@ """Test stack versioned schemas.""" import copy +import os import unittest +import unittest.mock import uuid from pathlib import Path import eql import pytest -import pytoml from marshmallow import ValidationError from semver import Version @@ -318,7 +319,7 @@ def test_esql_data_validation(self): # A random ESQL rule to deliver a test query rule_path = Path("tests/data/command_control_dummy_production_rule.toml") rule_body = rule_path.read_text() - rule_dict = pytoml.loads(rule_body) + rule_dict = RuleCollection.deserialize_toml_string(rule_body) # Most used order of the metadata fields query = """ @@ -357,3 +358,49 @@ def test_esql_data_validation(self): """ rule_dict["rule"]["query"] = query _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + def test_esql_keep_validation_bypass_missing_keep(self): + """ES|QL keep checks are skipped when DR_BYPASS_ESQL_KEEP_VALIDATION is set.""" + # A random ESQL rule to deliver a test query + rule_path = Path("tests/data/command_control_dummy_production_rule.toml") + rule_body = rule_path.read_text() + rule_dict = RuleCollection.deserialize_toml_string(rule_body) + query = """ + FROM logs-windows.powershell_operational* METADATA _id, _index, _version + | WHERE event.code == "4104" + """ + rule_dict["rule"]["query"] = query + with unittest.mock.patch.dict(os.environ, {"DR_BYPASS_ESQL_KEEP_VALIDATION": "1"}): + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + def test_esql_keep_bypass_does_not_skip_from_metadata_validation(self): + """FROM METADATA requirement still applies when only keep validation is bypassed.""" + # A random ESQL rule to deliver a test query + rule_path = Path("tests/data/command_control_dummy_production_rule.toml") + rule_body = rule_path.read_text() + rule_dict = RuleCollection.deserialize_toml_string(rule_body) + query = """ + FROM logs-windows.powershell_operational* + | WHERE event.code == "4104" + """ + rule_dict["rule"]["query"] = query + with ( + unittest.mock.patch.dict(os.environ, {"DR_BYPASS_ESQL_KEEP_VALIDATION": "1"}), + pytest.raises(EsqlSemanticError), + ): + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + def test_esql_keep_validation_bypass_missing_metadata_in_keep(self): + """ES|QL metadata-in-keep checks are skipped when DR_BYPASS_ESQL_KEEP_VALIDATION is set.""" + # A random ESQL rule to deliver a test query + rule_path = Path("tests/data/command_control_dummy_production_rule.toml") + rule_body = rule_path.read_text() + rule_dict = RuleCollection.deserialize_toml_string(rule_body) + query = """ + FROM logs-windows.powershell_operational* METADATA _id, _version, _index + | WHERE event.code == "4104" + | KEEP event.code + """ + rule_dict["rule"]["query"] = query + with unittest.mock.patch.dict(os.environ, {"DR_BYPASS_ESQL_KEEP_VALIDATION": "1"}): + _ = RuleCollection().load_dict(rule_dict, path=rule_path) From 63e3f797eb368ee7452beba54011e943b86b2f3d Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 13:10:48 -0400 Subject: [PATCH 2/6] Add metadata bypass --- CLI.md | 2 ++ detection_rules/rule.py | 16 +++++++++++----- detection_rules/utils.py | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CLI.md b/CLI.md index e331b86272d..c46ec62fef4 100644 --- a/CLI.md +++ b/CLI.md @@ -48,6 +48,8 @@ Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will byp Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass local validation that ES|QL rules include a `keep` command and that non-aggregate queries list `_id`, `_version`, and `_index` in `keep` (other ES|QL checks are unchanged). +Using the environment variable `DR_BYPASS_ESQL_METADATA_VALIDATION` will bypass local validation that non-aggregate ES|QL queries use `FROM ... METADATA _id, _version, _index` or an aggregate `STATS ... BY` pattern (other ES|QL checks are unchanged). + Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI. For instance, some users may want to increase the default value in cases where help messages are cut off. diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 11a3e5dd76f..5afff9b012a 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -981,12 +981,18 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None: ) # Ensure that non-aggregate queries have metadata - if not combined_pattern.search(query_lower): - raise EsqlSemanticError( - f"Rule: {data['name']} contains a non-aggregate query without" - f" metadata fields '_id', '_version', and '_index' ->" - f" Add 'metadata _id, _version, _index' to the from command or add an aggregate function." + if os.environ.get("DR_BYPASS_ESQL_METADATA_VALIDATION") is None: + bypass_metadata_hint = ( + " To bypass ES|QL `FROM` metadata validation, set the environment variable " + "`DR_BYPASS_ESQL_METADATA_VALIDATION`." ) + if not combined_pattern.search(query_lower): + raise EsqlSemanticError( + f"Rule: {data['name']} contains a non-aggregate query without" + f" metadata fields '_id', '_version', and '_index' ->" + f" Add 'metadata _id, _version, _index' to the from command or add an aggregate function." + + bypass_metadata_hint + ) # Enforce KEEP command for ESQL rules and that METADATA fields are present in non-aggregate queries if os.environ.get("DR_BYPASS_ESQL_KEEP_VALIDATION") is None: diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 916db9434c0..f59ca23d85b 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -143,6 +143,7 @@ def set_all_validation_bypass(env_value: bool = False) -> None: os.environ["DR_BYPASS_TAGS_VALIDATION"] = str(env_value) os.environ["DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION"] = str(env_value) os.environ["DR_BYPASS_ESQL_KEEP_VALIDATION"] = str(env_value) + os.environ["DR_BYPASS_ESQL_METADATA_VALIDATION"] = str(env_value) def set_nested_value(obj: dict[str, Any], compound_key: str, value: Any) -> None: From a2015caca4296d1b22e252dae15684af0487febf Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 13:11:10 -0400 Subject: [PATCH 3/6] Add unit test for metadata bypass --- tests/test_schemas.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index b365a0dcaa4..7fb913deaab 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -390,6 +390,38 @@ def test_esql_keep_bypass_does_not_skip_from_metadata_validation(self): ): _ = RuleCollection().load_dict(rule_dict, path=rule_path) + def test_esql_metadata_validation_bypass_missing_from_metadata(self): + """ES|QL FROM METADATA checks are skipped when DR_BYPASS_ESQL_METADATA_VALIDATION is set.""" + # A random ESQL rule to deliver a test query + rule_path = Path("tests/data/command_control_dummy_production_rule.toml") + rule_body = rule_path.read_text() + rule_dict = RuleCollection.deserialize_toml_string(rule_body) + query = """ + FROM logs-windows.powershell_operational* + | WHERE event.code == "4104" + | KEEP event.code, _id, _version, _index + """ + rule_dict["rule"]["query"] = query + with unittest.mock.patch.dict(os.environ, {"DR_BYPASS_ESQL_METADATA_VALIDATION": "1"}): + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + def test_esql_metadata_bypass_does_not_skip_keep_validation(self): + """`keep` validation still applies when only FROM metadata validation is bypassed.""" + # A random ESQL rule to deliver a test query + rule_path = Path("tests/data/command_control_dummy_production_rule.toml") + rule_body = rule_path.read_text() + rule_dict = RuleCollection.deserialize_toml_string(rule_body) + query = """ + FROM logs-windows.powershell_operational* + | WHERE event.code == "4104" + """ + rule_dict["rule"]["query"] = query + with ( + unittest.mock.patch.dict(os.environ, {"DR_BYPASS_ESQL_METADATA_VALIDATION": "1"}), + pytest.raises(EsqlSemanticError), + ): + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + def test_esql_keep_validation_bypass_missing_metadata_in_keep(self): """ES|QL metadata-in-keep checks are skipped when DR_BYPASS_ESQL_KEEP_VALIDATION is set.""" # A random ESQL rule to deliver a test query From 648fa58e725cc94267188b91c4bcc800a3faa046 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 13:11:32 -0400 Subject: [PATCH 4/6] Update docs --- docs-dev/custom-rules-management.md | 2 +- docs-dev/developing.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs-dev/custom-rules-management.md b/docs-dev/custom-rules-management.md index a9222e4c189..c6917a1d821 100644 --- a/docs-dev/custom-rules-management.md +++ b/docs-dev/custom-rules-management.md @@ -76,7 +76,7 @@ Some notes: * To manage action-connectors tied to rules one can set an action-connectors directory using the optional `action_connector_dir` value (included above) set to be the desired path. If an actions_connector directory is explicitly specified in a CLI command, the config value will be ignored. * To turn on automatic schema generation for non-ecs fields via custom schemas add `auto_gen_schema_file: `. This will generate a schema file in the specified location that will be used to add entries for each field and index combination that is not already in a known schema. This will also automatically add it to your stack-schema-map.yaml file when using a custom rules directory and config. * For Kibana action items, currently these are included in the rule toml files themselves. At a later date, we may allow for bulk editing of rule action items through separate action toml files. The action_dir config key is left available for this later implementation. For now to bulk update, use the bulk actions add rule actions UI in Kibana. -* To on bulk disable elastic validation for optional fields, use the following line `bypass_optional_elastic_validation: True`. That sets the shared `DR_BYPASS_*` toggles (including note, BBR lookback, tags, timeline template, and ES|QL `keep` validation). +* To on bulk disable elastic validation for optional fields, use the following line `bypass_optional_elastic_validation: True`. That sets the shared `DR_BYPASS_*` toggles (including note, BBR lookback, tags, timeline template, and ES|QL `keep` / `FROM` metadata validation). When using the repo, set the environment variable `CUSTOM_RULES_DIR=` diff --git a/docs-dev/developing.md b/docs-dev/developing.md index 3cf3ef5bff5..9cbd86fe512 100644 --- a/docs-dev/developing.md +++ b/docs-dev/developing.md @@ -50,6 +50,8 @@ Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will byp Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass local validation that ES|QL rules include a `keep` command and that non-aggregate queries list `_id`, `_version`, and `_index` in `keep` (other ES|QL checks are unchanged). +Using the environment variable `DR_BYPASS_ESQL_METADATA_VALIDATION` will bypass local validation that non-aggregate ES|QL queries use `FROM ... METADATA _id, _version, _index` or an aggregate `STATS ... BY` pattern (other ES|QL checks are unchanged). + ## Using the `RuleResource` methods built on detections `_bulk_action` APIs From baf265de132ee8b9da3423f623ebaeee331e7ea0 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 15:01:57 -0400 Subject: [PATCH 5/6] Added additional config support --- CLI.md | 2 ++ detection_rules/config.py | 31 +++++++++++++++++++++++++++-- detection_rules/etc/_config.yaml | 16 +++++++++++++-- detection_rules/utils.py | 19 ++++++++++++------ docs-dev/custom-rules-management.md | 9 ++++++++- docs-dev/developing.md | 2 ++ 6 files changed, 68 insertions(+), 11 deletions(-) diff --git a/CLI.md b/CLI.md index c46ec62fef4..37960b936f7 100644 --- a/CLI.md +++ b/CLI.md @@ -50,6 +50,8 @@ Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass loca Using the environment variable `DR_BYPASS_ESQL_METADATA_VALIDATION` will bypass local validation that non-aggregate ES|QL queries use `FROM ... METADATA _id, _version, _index` or an aggregate `STATS ... BY` pattern (other ES|QL checks are unchanged). +In `_config.yaml`, `bypass_optional_elastic_validation: true` enables all of the above at load time. Alternatively, set any of the top-level booleans `bypass_note_validation_and_parse`, `bypass_bbr_lookback_validation`, `bypass_tags_validation`, `bypass_timeline_template_validation`, `bypass_esql_keep_validation`, or `bypass_esql_metadata_validation` to `true` (see comments in `detection_rules/etc/_config.yaml`). + Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI. For instance, some users may want to increase the default value in cases where help messages are cut off. diff --git a/detection_rules/config.py b/detection_rules/config.py index 33be0050b95..c03d2837041 100644 --- a/detection_rules/config.py +++ b/detection_rules/config.py @@ -16,7 +16,13 @@ from eql.utils import load_dump # type: ignore[reportMissingTypeStubs] from .misc import discover_tests -from .utils import cached, get_etc_path, load_etc_dump, set_all_validation_bypass +from .utils import ( + OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV, + cached, + get_etc_path, + load_etc_dump, + set_all_validation_bypass, +) ROOT_DIR = Path(__file__).parent.parent CUSTOM_RULES_DIR = os.getenv("CUSTOM_RULES_DIR", None) @@ -208,6 +214,12 @@ class RulesConfig: exception_dir: Path | None = None normalize_kql_keywords: bool = True bypass_optional_elastic_validation: bool = False + bypass_note_validation_and_parse: bool = False + bypass_bbr_lookback_validation: bool = False + bypass_tags_validation: bool = False + bypass_timeline_template_validation: bool = False + bypass_esql_keep_validation: bool = False + bypass_esql_metadata_validation: bool = False no_tactic_filename: bool = False def __post_init__(self) -> None: @@ -323,7 +335,22 @@ def parse_rules_config(path: Path | None = None) -> RulesConfig: # noqa: PLR091 # bypass_optional_elastic_validation contents["bypass_optional_elastic_validation"] = loaded.get("bypass_optional_elastic_validation", False) if contents["bypass_optional_elastic_validation"]: - set_all_validation_bypass(contents["bypass_optional_elastic_validation"]) + set_all_validation_bypass(True) + for yaml_key in OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV: + contents[yaml_key] = True + else: + for yaml_key, env_var in OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV.items(): + if yaml_key in loaded: + val = loaded[yaml_key] + if not isinstance(val, bool): + raise SystemExit( + f"`{yaml_key}` in _config.yaml must be a boolean (true/false), not {type(val).__name__}" + ) + else: + val = False + contents[yaml_key] = val + if val: + os.environ[env_var] = str(True) # no_tactic_filename contents["no_tactic_filename"] = loaded.get("no_tactic_filename", False) diff --git a/detection_rules/etc/_config.yaml b/detection_rules/etc/_config.yaml index 08377486ff1..2cb9c1441a2 100644 --- a/detection_rules/etc/_config.yaml +++ b/detection_rules/etc/_config.yaml @@ -61,8 +61,20 @@ normalize_kql_keywords: False # stack-schema-map.yaml file when using a custom rules directory and config. # auto_gen_schema_file: "etc/auto-gen-schema.json" -# To on bulk disable elastic validation for optional fields, use the following line -# bypass_optional_elastic_validation: True +# Optional Elastic validation bypasses (each true value sets the matching DR_BYPASS_* env var at load time). +# +# 1) Enable every bypass at once: +# bypass_optional_elastic_validation: true +# +# 2) Or set only the bypasses you need (ignored if bypass_optional_elastic_validation is true): +# bypass_note_validation_and_parse: true # DR_BYPASS_NOTE_VALIDATION_AND_PARSE +# bypass_bbr_lookback_validation: true # DR_BYPASS_BBR_LOOKBACK_VALIDATION +# bypass_tags_validation: true # DR_BYPASS_TAGS_VALIDATION +# bypass_timeline_template_validation: true # DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION +# bypass_esql_keep_validation: true # DR_BYPASS_ESQL_KEEP_VALIDATION +# bypass_esql_metadata_validation: true # DR_BYPASS_ESQL_METADATA_VALIDATION +# +# Each must be true or false if present; omitted keys default to false. # This points to the testing config file (see example under detection_rules/etc/example_test_config.yaml) # This can either be set here or as the environment variable `DETECTION_RULES_TEST_CONFIG`, with precedence diff --git a/detection_rules/utils.py b/detection_rules/utils.py index f59ca23d85b..80c863b8a8b 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -136,14 +136,21 @@ def save_etc_dump(contents: dict[str, Any], path: list[str], sort_keys: bool = T eql.utils.save_dump(contents, path) # type: ignore[reportUnknownVariableType] +# Top-level _config.yaml key -> DR_BYPASS_* env var set when true at load time +OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV: dict[str, str] = { + "bypass_note_validation_and_parse": "DR_BYPASS_NOTE_VALIDATION_AND_PARSE", + "bypass_bbr_lookback_validation": "DR_BYPASS_BBR_LOOKBACK_VALIDATION", + "bypass_tags_validation": "DR_BYPASS_TAGS_VALIDATION", + "bypass_timeline_template_validation": "DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION", + "bypass_esql_keep_validation": "DR_BYPASS_ESQL_KEEP_VALIDATION", + "bypass_esql_metadata_validation": "DR_BYPASS_ESQL_METADATA_VALIDATION", +} + + def set_all_validation_bypass(env_value: bool = False) -> None: """Set all validation bypass environment variables.""" - os.environ["DR_BYPASS_NOTE_VALIDATION_AND_PARSE"] = str(env_value) - os.environ["DR_BYPASS_BBR_LOOKBACK_VALIDATION"] = str(env_value) - os.environ["DR_BYPASS_TAGS_VALIDATION"] = str(env_value) - os.environ["DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION"] = str(env_value) - os.environ["DR_BYPASS_ESQL_KEEP_VALIDATION"] = str(env_value) - os.environ["DR_BYPASS_ESQL_METADATA_VALIDATION"] = str(env_value) + for env_var in OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV.values(): + os.environ[env_var] = str(env_value) def set_nested_value(obj: dict[str, Any], compound_key: str, value: Any) -> None: diff --git a/docs-dev/custom-rules-management.md b/docs-dev/custom-rules-management.md index c6917a1d821..5c8bebb730e 100644 --- a/docs-dev/custom-rules-management.md +++ b/docs-dev/custom-rules-management.md @@ -76,7 +76,8 @@ Some notes: * To manage action-connectors tied to rules one can set an action-connectors directory using the optional `action_connector_dir` value (included above) set to be the desired path. If an actions_connector directory is explicitly specified in a CLI command, the config value will be ignored. * To turn on automatic schema generation for non-ecs fields via custom schemas add `auto_gen_schema_file: `. This will generate a schema file in the specified location that will be used to add entries for each field and index combination that is not already in a known schema. This will also automatically add it to your stack-schema-map.yaml file when using a custom rules directory and config. * For Kibana action items, currently these are included in the rule toml files themselves. At a later date, we may allow for bulk editing of rule action items through separate action toml files. The action_dir config key is left available for this later implementation. For now to bulk update, use the bulk actions add rule actions UI in Kibana. -* To on bulk disable elastic validation for optional fields, use the following line `bypass_optional_elastic_validation: True`. That sets the shared `DR_BYPASS_*` toggles (including note, BBR lookback, tags, timeline template, and ES|QL `keep` / `FROM` metadata validation). +* To disable optional Elastic validation in bulk, set `bypass_optional_elastic_validation: true` in `_config.yaml`. That sets every `DR_BYPASS_*` environment variable that `set_all_validation_bypass()` controls (note parsing, BBR lookback, tags unit tests, timeline template, ES|QL `keep`, ES|QL `FROM` metadata). +* To enable only some of those bypasses, set the matching top-level booleans in `_config.yaml` (omit `bypass_optional_elastic_validation` or set it to `false`): `bypass_note_validation_and_parse`, `bypass_bbr_lookback_validation`, `bypass_tags_validation`, `bypass_timeline_template_validation`, `bypass_esql_keep_validation`, `bypass_esql_metadata_validation`. Each `true` sets the corresponding `DR_BYPASS_*` variable when the config is loaded. If `bypass_optional_elastic_validation` is `true`, those individual flags are all treated as enabled (the bulk flag wins). When using the repo, set the environment variable `CUSTOM_RULES_DIR=` @@ -132,6 +133,12 @@ class RulesConfig: exception_dir: Optional[Path] = None normalize_kql_keywords: bool = True bypass_optional_elastic_validation: bool = False + bypass_note_validation_and_parse: bool = False + bypass_bbr_lookback_validation: bool = False + bypass_tags_validation: bool = False + bypass_timeline_template_validation: bool = False + bypass_esql_keep_validation: bool = False + bypass_esql_metadata_validation: bool = False # using the stack_schema_map RULES_CONFIG.stack_schema_map diff --git a/docs-dev/developing.md b/docs-dev/developing.md index 9cbd86fe512..6a3563afbee 100644 --- a/docs-dev/developing.md +++ b/docs-dev/developing.md @@ -52,6 +52,8 @@ Using the environment variable `DR_BYPASS_ESQL_KEEP_VALIDATION` will bypass loca Using the environment variable `DR_BYPASS_ESQL_METADATA_VALIDATION` will bypass local validation that non-aggregate ES|QL queries use `FROM ... METADATA _id, _version, _index` or an aggregate `STATS ... BY` pattern (other ES|QL checks are unchanged). +In `_config.yaml`, `bypass_optional_elastic_validation: true` enables all of these bypass env vars when config is loaded. You can instead set individual top-level flags (`bypass_note_validation_and_parse`, `bypass_bbr_lookback_validation`, `bypass_tags_validation`, `bypass_timeline_template_validation`, `bypass_esql_keep_validation`, `bypass_esql_metadata_validation`); the bulk flag takes precedence if it is true. See `detection_rules/etc/_config.yaml` for an example. + ## Using the `RuleResource` methods built on detections `_bulk_action` APIs From e221e09a305583ddd416cf2e89dc256922ebef44 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Mon, 23 Mar 2026 16:01:53 -0400 Subject: [PATCH 6/6] Patch bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e030095472..73358dd11a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.7" +version = "1.6.8" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"