From 2915d369f0ab8e7d3cf5d8b0eccbae0e7b9d96c6 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:40:06 -0500 Subject: [PATCH 01/14] Add support for AWS enrichment --- .../protocol_logs/corelight-ecs-conn-pipeline | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline index d500af6..0a5c01a 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline @@ -281,6 +281,146 @@ "ignore_missing": true } }, + { + "rename": { + "field": "resp_inst.az", + "target_field": "conn.resp_inst.az", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.id", + "target_field": "conn.resp_inst.id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.name", + "target_field": "conn.resp_inst.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.orig_id", + "target_field": "conn.resp_inst.orig_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.vpc_id", + "target_field": "conn.resp_inst.vpc_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.az", + "target_field": "conn.resp_inst.az", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.id", + "target_field": "conn.resp_inst.id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.name", + "target_field": "conn.resp_inst.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.orig_id", + "target_field": "conn.resp_inst.orig_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.vpc_id", + "target_field": "conn.resp_inst.vpc_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.az", + "target_field": "conn.resp_inst.az", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.id", + "target_field": "conn.resp_inst.id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.name", + "target_field": "conn.resp_inst.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.orig_id", + "target_field": "conn.resp_inst.orig_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "resp_inst.vpc_id", + "target_field": "conn.resp_inst.vpc_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "orig_inst.az", + "target_field": "conn.orig_inst.az", + "ignore_missing": true + } + }, + { + "rename": { + "field": "orig_inst.id", + "target_field": "conn.orig_inst.id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "orig_inst.name", + "target_field": "conn.orig_inst.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "orig_inst.orig_id", + "target_field": "conn.orig_inst.orig_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "orig_inst.vpc_id", + "target_field": "conn.orig_inst.vpc_id", + "ignore_missing": true + } + }, { "rename": { "field": "corelight_shunted", From 92397c0f9189e7cdb5d9789cb245cd68835c0c40 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:34:37 -0600 Subject: [PATCH 02/14] Update ssh inferences ssh inferences were not being mapped to the ecs as ssh.inferences this has been fixed. --- .../log_specific/protocol_logs/corelight-ecs-ssh-pipeline | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-ssh-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-ssh-pipeline index 4024d12..5d4e947 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-ssh-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-ssh-pipeline @@ -222,6 +222,13 @@ "ignore_missing": true } }, + { + "rename": { + "field": "inferences", + "target_field": "ssh.inferences", + "ignore_missing": true + } + }, { "set": { "field": "network.transport", From ecb919c42ba80aa395bf568ff4feadb359c13951 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Fri, 31 Jan 2025 09:07:55 -0600 Subject: [PATCH 03/14] DNS/Conn Agg pipeline add support for DNS/Conn Agg pipeline --- .../protocol_logs/corelight-ecs-conn_agg-pipeline | 11 +++++++++++ .../protocol_logs/corelight-ecs-dns_agg-pipeline | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-conn_agg-pipeline create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-dns_agg-pipeline diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-conn_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-conn_agg-pipeline new file mode 100644 index 0000000..b0c3b83 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-conn_agg-pipeline @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'conn_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025013001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-conn-pipeline" + } + } + ] +} diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-dns_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-dns_agg-pipeline new file mode 100644 index 0000000..8844355 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-dns_agg-pipeline @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'dns_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025013001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-dns-pipeline" + } + } + ] + } \ No newline at end of file From eb6b54fba6689d6396418d5593d9059e2c26c8d6 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Fri, 31 Jan 2025 09:24:27 -0600 Subject: [PATCH 04/14] DNS/Conn Agg pipeline adqd support for DNS/Conn Agg Pipeline --- .../log_specific/protocol_logs/corelight-ecs-conn-pipeline | 7 +++++++ .../log_specific/protocol_logs/corelight-ecs-dns-pipeline | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline index 0a5c01a..9284d52 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline @@ -680,6 +680,13 @@ "ignore_missing": true } }, + { + "rename": { + "field": "ts_last", + "target_field": "corelight.ts_last", + "ignore_missing": true + } + }, { "remove": { "field": [ "id", "id_vlan" ], diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-dns-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-dns-pipeline index c02e5c3..2bf13b8 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-dns-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-dns-pipeline @@ -263,6 +263,13 @@ "ignore_failure": true } }, + { + "rename": { + "field": "ts_last", + "target_field": "corelight.ts_last", + "ignore_missing": true + } + }, { "append": { "field": "dns.header_flags", From 414a182cf8a8d4b279135e69cf2b89a111d2b71d Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:26:28 -0500 Subject: [PATCH 05/14] Added Wierd_agg support and fix Auditlogs --- pipeline/corelight-ecs-common-main-pipeline | 11 +++++++++++ .../protocol_logs/corelight-ecs-wierd_agg-pipeline | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline diff --git a/pipeline/corelight-ecs-common-main-pipeline b/pipeline/corelight-ecs-common-main-pipeline index b0a03c1..6561aa5 100644 --- a/pipeline/corelight-ecs-common-main-pipeline +++ b/pipeline/corelight-ecs-common-main-pipeline @@ -111,6 +111,17 @@ "if": "ctx['@stream'] != null" } }, + { + "date": { + "field": "time", + "target_field": "ts", + "formats": [ + "ISO8601", + "UNIX" + ], + "if": "ctx?.event.dataset == 'auditlog'" + } + }, { "date": { "field": "ts", diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline new file mode 100644 index 0000000..e83fc07 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'weird_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025061001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-weird-pipeline" + } + } + ] +} \ No newline at end of file From 6b352e6e8a55c67b1e8331f70320ba7078071c57 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Thu, 12 Jun 2025 10:44:40 -0500 Subject: [PATCH 06/14] Rename pipeline to fix name --- ...ht-ecs-wierd_agg-pipeline => corelight-ecs-weird_agg-pipeline} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pipeline/log_specific/protocol_logs/{corelight-ecs-wierd_agg-pipeline => corelight-ecs-weird_agg-pipeline} (100%) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-weird_agg-pipeline similarity index 100% rename from pipeline/log_specific/protocol_logs/corelight-ecs-wierd_agg-pipeline rename to pipeline/log_specific/protocol_logs/corelight-ecs-weird_agg-pipeline From c68e1302c420f7e086ffec407bc092e6c7246f67 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:02:55 -0500 Subject: [PATCH 07/14] ecs-69 Add support for Anomaly logs --- .../corelight-ecs-anomaly-pipeline | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline new file mode 100644 index 0000000..6654148 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline @@ -0,0 +1,136 @@ +{ + "description": "Corelight ingest pipeline for 'anomaly' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025061301, + "processors": [ + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.category", + "value": "network", + "override": true + } + }, + { + "set": { + "field": "event.type", + "value": "info" + } + }, + { + "set": { + "field": "labels.corelight.event_category", + "value": "miscellaneous" + } + }, + { + "set": { + "field": "temporary_metadata_index_name_type", + "value": "VAR_CL_DS_TYPE_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_dataset_prefix", + "value": "VAR_CL_DS_PREFIX_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_dataset_suffix", + "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_WEIRD", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_namespace", + "value": "VAR_CL_DS_NAMESPACE_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "set": { + "field": "usecase", + "value": "anomaly.usecase", + "ignore_failure": false + } + }, + { + "set": { + "field": "usecase_description", + "value": "anomaly.usecase_description", + "ignore_failure": false + } + }, + { + "set": { + "field": "entity", + "value": "anomaly.entity", + "ignore_failure": false + } + }, + { + "set": { + "field": "original_entity", + "value": "anomaly.original_entity", + "ignore_failure": false + } + }, + { + "set": { + "field": "entity_training_items", + "value": "anomaly.entity_training_items", + "ignore_failure": false + } + }, + { + "set": { + "field": "item_assoc_entities", + "value": "anomaly.item_assoc_entities", + "ignore_failure": false + } + }, + { + "set": { + "field": "item_score", + "value": "anomaly.item_score", + "ignore_failure": false + } + }, + { + "set": { + "field": "item_assoc_entities", + "value": "anomaly.item_assoc_entities", + "ignore_failure": false + } + }, + { + "set": { + "field": "item_assoc_entities_similarity", + "value": "anomaly.item_assoc_entities_similarity", + "ignore_failure": false + } + }, + { + "set": { + "field": "ignorable", + "value": "anomaly.ignorable", + "ignore_failure": false + } + }, + { + "set": { + "field": "anomaly_type", + "value": "anomaly.anomaly_type", + "ignore_failure": false + } + } + ] + } \ No newline at end of file From de6421298c6e771cdfca4786c72f64f235c7276a Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:47:55 -0500 Subject: [PATCH 08/14] Change dataset --- .../log_specific/protocol_logs/corelight-ecs-anomaly-pipeline | 2 +- .../log_specific/protocol_logs/corelight-ecs-telnet-pipeline | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline index 6654148..472d1fb 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-anomaly-pipeline @@ -44,7 +44,7 @@ { "set": { "field": "temporary_metadata_index_name_dataset_suffix", - "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_WEIRD", + "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_VARIOUS", "ignore_failure": false } }, diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-telnet-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-telnet-pipeline index 10f2286..dd25534 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-telnet-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-telnet-pipeline @@ -44,7 +44,7 @@ { "set": { "field": "temporary_metadata_index_name_dataset_suffix", - "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_DNS", + "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_VARIOUS", "ignore_failure": false } }, From 9e7ee6f3fa512ca12f7d7947081652aa62c459fd Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:39:43 -0500 Subject: [PATCH 09/14] ECS-76 Add support for corelight_service_status --- pipeline/corelight-ecs-main-pipeline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/corelight-ecs-main-pipeline b/pipeline/corelight-ecs-main-pipeline index 4d06af4..3b391d3 100644 --- a/pipeline/corelight-ecs-main-pipeline +++ b/pipeline/corelight-ecs-main-pipeline @@ -26,7 +26,7 @@ "description": "Common pipeline for Metric logs (metrics and stats logs).", "name": "corelight-ecs-common-metric-metrics_stats-pipeline", "ignore_missing_pipeline": false, - "if": "ctx?.labels?.corelight?.event_sub_type != null && [ 'capture_loss', 'conn_doctor', 'corelight_cloud_stats', 'corelight_metrics_bro', 'corelight_metrics_cpu', 'corelight_metrics_disk', 'corelight_metrics_docker', 'corelight_metrics_iface', 'corelight_metrics_memory', 'corelight_metrics_s3', 'corelight_metrics_sftp', 'corelight_metrics_suricata', 'corelight_metrics_system', 'corelight_metrics_utilization', 'corelight_ml_metrics', 'corelight_overall_capture_loss', 'corelight_profiling', 'corelight_weird_stats', 'ml_metrics', 'namecache', 'packet_filter', 'reporter', 'smartpcap-stats', 'stats', 'suricata_stats', 'weird_stats' ].contains(ctx.labels?.corelight?.event_sub_type)" + "if": "ctx?.labels?.corelight?.event_sub_type != null && [ 'capture_loss', 'conn_doctor', 'corelight_cloud_stats', 'corelight_metrics_bro', 'corelight_metrics_cpu', 'corelight_metrics_disk', 'corelight_metrics_docker', 'corelight_metrics_iface', 'corelight_metrics_memory', 'corelight_metrics_s3', 'corelight_metrics_sftp', 'corelight_metrics_suricata', 'corelight_metrics_system', 'corelight_metrics_utilization', 'corelight_ml_metrics', 'corelight_overall_capture_loss', 'corelight_profiling', 'corelight_service_status', 'corelight_weird_stats', 'ml_metrics', 'namecache', 'packet_filter', 'reporter', 'smartpcap-stats', 'stats', 'suricata_stats', 'weird_stats' ].contains(ctx.labels?.corelight?.event_sub_type)" } }, { From cc757367c2c4a9bca3fd896fddf425821fb3156f Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:14:46 -0600 Subject: [PATCH 10/14] fix x509 and add support for Suricata Payload --- .gitignore | 1 + .../corelight-ecs-suricata_corelight-pipeline | 7 ++++ .../protocol_logs/corelight-ecs-x509-pipeline | 34 +++++++++---------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 9bea433..7274eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store +pipeline/log_specific/protocol_logs/corelight-ecs-investigator_alerts-pipeline.json diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-suricata_corelight-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-suricata_corelight-pipeline index 2e1d214..8e14667 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-suricata_corelight-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-suricata_corelight-pipeline @@ -328,6 +328,13 @@ "ignore_missing": true } }, + { + "rename": { + "field": "payload", + "target_field": "suricata.payload", + "ignore_missing": true + } + }, { "set": { "field": "threat.tactic.reference", diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-x509-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-x509-pipeline index 50ec3ee..ea55cdf 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-x509-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-x509-pipeline @@ -181,7 +181,7 @@ { "rename": { "field": "certificate.not_valid_after", - "target_field": "file.x509.not_after", + "target_field": "file.x509.not_after_non_formatted_date", "ignore_missing": true } }, @@ -194,7 +194,7 @@ { "rename": { "field": "certificate.not_valid_before", - "target_field": "file.x509.not_before", + "target_field": "file.x509.not_before_non_formatted_date", "ignore_missing": true } }, @@ -393,18 +393,11 @@ "ignore_missing": true } }, - { - "rename": { - "field": "certificate_not_valid_after", - "target_field": "file.x509.not_after_non_formatted_date", - "ignore_missing": true - } - }, { "date": { "field": "file.x509.not_after_non_formatted_date", "target_field": "file.x509.not_after", - "formats": [ "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ", "UNIX", "YYYY-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" ], + "formats": [ "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX", "UNIX", "YYYY-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" ], "on_failure": [ { "remove": { @@ -415,18 +408,11 @@ ] } }, - { - "rename": { - "field": "certificate_not_valid_before", - "target_field": "file.x509.not_before_non_formatted_date", - "ignore_missing": true - } - }, { "date": { "field": "file.x509.not_before_non_formatted_date", "target_field": "file.x509.not_before", - "formats": [ "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ", "UNIX", "YYYY-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" ], + "formats": [ "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX", "UNIX", "YYYY-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" ], "on_failure": [ { "remove": { @@ -554,6 +540,18 @@ "ignore_failure": true, "ignore_missing": true } + }, + { + "remove": { + "field": "file.x509.not_before_non_formatted_date", + "if": "ctx.file?.x509?.not_before != null" + } + }, + { + "remove": { + "field": "file.x509.not_after_non_formatted_date", + "if": "ctx.file?.x509?.not_after != null" + } } ] } From 08a94edfa93518bafd51d29f3fe1f670a00cb9e0 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Wed, 10 Dec 2025 08:53:15 -0600 Subject: [PATCH 11/14] ECS-80 --- .../protocol_logs/corelight-ecs-files_agg-pipeline | 11 +++++++++++ .../protocol_logs/corelight-ecs-http_agg-pipeline | 11 +++++++++++ .../protocol_logs/corelight-ecs-ssl_agg-pipeline.json | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-files_agg-pipeline create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-http_agg-pipeline create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-ssl_agg-pipeline.json diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-files_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-files_agg-pipeline new file mode 100644 index 0000000..66ca56f --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-files_agg-pipeline @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'files_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025013001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-files-pipeline" + } + } + ] + } \ No newline at end of file diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-http_agg-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-http_agg-pipeline new file mode 100644 index 0000000..78c1454 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-http_agg-pipeline @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'http_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025013001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-http-pipeline" + } + } + ] + } \ No newline at end of file diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-ssl_agg-pipeline.json b/pipeline/log_specific/protocol_logs/corelight-ecs-ssl_agg-pipeline.json new file mode 100644 index 0000000..efc231f --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-ssl_agg-pipeline.json @@ -0,0 +1,11 @@ +{ + "description": "Corelight ingest pipeline for 'ssl_agg' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025013001, + "processors": [ + { + "pipeline": { + "name": "corelight-ecs-ssl-pipeline" + } + } + ] + } \ No newline at end of file From 9c20dd6ad75bf42ba5bbc4848a5e1af3904ec2d9 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Tue, 16 Dec 2025 10:17:14 -0600 Subject: [PATCH 12/14] ECS-81 Support for unknown logs --- .../corelight-ecs-unknown_protocols-pipeline | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 pipeline/log_specific/protocol_logs/corelight-ecs-unknown_protocols-pipeline diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-unknown_protocols-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-unknown_protocols-pipeline new file mode 100644 index 0000000..758bbb0 --- /dev/null +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-unknown_protocols-pipeline @@ -0,0 +1,94 @@ +{ + "description": "Corelight ingest pipeline for 'unknown_protocols' log. This ingest pipeline is from the Github repository https://github.com/corelight/ecs-mapping. Please file all questions or issues with this configuration in the corresponding Github repository.", + "version": 2025121501, + "processors":[ + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.category", + "value": "network", + "override": true + } + }, + { + "set": { + "field": "event.type", + "value": [ "connection", "info", "protocol" ] + } + }, + { + "set": { + "field": "labels.corelight.event_category", + "value": "network_protocols" + } + }, + { + "set": { + "field": "temporary_metadata_index_name_type", + "value": "VAR_CL_DS_TYPE_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_dataset_prefix", + "value": "VAR_CL_DS_PREFIX_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_dataset_suffix", + "value": "VAR_CL_DS_SUFFIX_PROTOCOL_LOG_VARIOUS", + "ignore_failure": false + } + }, + { + "set": { + "field": "temporary_metadata_index_name_namespace", + "value": "VAR_CL_DS_NAMESPACE_PROTOCOL_LOG", + "ignore_failure": false + } + }, + { + "rename": { + "field": "analyzer", + "target_field": "network.unknown.protocol", + "ignore_missing": true + } + }, + { + "rename": { + "field": "analyzer_history", + "target_field": "network.unknown.protocol_history", + "ignore_missing": true + } + }, + { + "rename": { + "field": "protocol_id", + "target_field": "network.unknown.protocol_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "first_bytes", + "target_field": "network.unknown.first_bytes", + "ignore_missing": true + } + }, + { + "rename": { + "field": "protocol_name", + "target_field": "network.unknown.protocol_name", + "ignore_missing": true + } + } +] +} \ No newline at end of file From 9934a42d3dd6a77c342dfe4b10525edfe771702e Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:48:54 -0500 Subject: [PATCH 13/14] Agg_update --- pipeline/corelight-ecs-common-main-pipeline | 7 +++++++ .../log_specific/protocol_logs/corelight-ecs-conn-pipeline | 7 +++++++ .../protocol_logs/corelight-ecs-files-pipeline | 7 +++++++ .../log_specific/protocol_logs/corelight-ecs-http-pipeline | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/pipeline/corelight-ecs-common-main-pipeline b/pipeline/corelight-ecs-common-main-pipeline index 6561aa5..debb925 100644 --- a/pipeline/corelight-ecs-common-main-pipeline +++ b/pipeline/corelight-ecs-common-main-pipeline @@ -403,6 +403,13 @@ "if": "ctx?.uid != null" } }, + { + "set": { + "field": "log.id.uids", + "value": "{{{uids}}}", + "if": "ctx?.uids != null" + } + }, { "rename": { "field": "@sensor", diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline index 9284d52..b518d9a 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline @@ -519,6 +519,13 @@ "ignore_missing": true } }, + { + "set": { + "field": "network.icmp_type", + "copy_from": "source.port", + "if": "(ctx['network.transport'].contains('icmp'))" + } + }, { "remove": { "field": [ diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-files-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-files-pipeline index cab7579..9e657bf 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-files-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-files-pipeline @@ -259,6 +259,13 @@ "target_field": "file.mime_type", "ignore_missing": true } + }, + { + "rename": { + "field": "mime_types", + "target_field": "file.mime_types", + "ignore_missing": true + } }, { "rename": { diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-http-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-http-pipeline index 2a6b8ac..8d98ac6 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-http-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-http-pipeline @@ -263,6 +263,13 @@ "target_field": "user_agent.original", "ignore_missing": true } + }, + { + "rename": { + "field": "user_agents", + "target_field": "user_agents.original", + "ignore_missing": true + } }, { "rename": { From 91b569d1db3b99f8bb4cd09749897eb4b0a10627 Mon Sep 17 00:00:00 2001 From: hunter32me <57907467+hunter32me@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:28:53 -0500 Subject: [PATCH 14/14] fixed contains error --- pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline index b518d9a..40cf6a2 100644 --- a/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline +++ b/pipeline/log_specific/protocol_logs/corelight-ecs-conn-pipeline @@ -523,7 +523,7 @@ "set": { "field": "network.icmp_type", "copy_from": "source.port", - "if": "(ctx['network.transport'].contains('icmp'))" + "if": "ctx?.network.transport == 'icmp'" } }, {