diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..14c1d8c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,6 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true diff --git a/.env.sentinel.example b/.env.sentinel.example new file mode 100644 index 0000000..90ed1d7 --- /dev/null +++ b/.env.sentinel.example @@ -0,0 +1,14 @@ +# Copiar a .env.sentinel y completar con las salidas del despliegue Azure CLI. +# No agregar .env.sentinel al repositorio. + +AZURE_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_CLIENT_SECRET= + +SENTINEL_DCE_ENDPOINT= +SENTINEL_DCR_IMMUTABLE_ID= +SENTINEL_STREAM_NAME=Custom-CloudSIEM + +SENTINEL_POLL_SECONDS=10 +SENTINEL_BATCH_SIZE=500 +SENTINEL_LOOKBACK_MINUTES=15 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/AGENTS.md b/.github/AGENTS.md new file mode 100644 index 0000000..db64d03 --- /dev/null +++ b/.github/AGENTS.md @@ -0,0 +1,40 @@ +# AGENTS.md + +## Repo Shape +- Read `README.md` and `.github/copilot-instructions.md` first; they define the project scope, required detections, and documentation language. +- The stack lives at the repository root; do not assume a nested `siem-elk/` directory. +- `develop` is the working branch; `main` is stable and PR-only. + +## Run The Stack +- For Podman on Linux, use `bash setup/run-podman-sentinel-stack.sh --build --force-recreate --no-sentinel` for the local stack, or remove `--no-sentinel` when Sentinel is enabled. +- Stop the matching stack with `podman-compose -f docker-compose.yml -f docker-compose-podman.yml down`, or add `-f docker-compose-sentinel.yml -f docker-compose-sentinel-podman.yml` for Sentinel. +- `docker-compose.yml` is the universal base; `docker-compose-podman.yml` is the Linux + Podman override and owns SELinux relabeling. +- The Podman override maps `$XDG_RUNTIME_DIR/podman/podman.sock` to `/var/run/docker.sock` for Filebeat metadata enrichment. +- The cluster bootstrap takes about 90 seconds before health checks are meaningful. + +## Verify +- From the repository root: `bash setup/verify-cluster.sh`, `python3 setup/check-cluster-health.py`, `python3 setup/verify-ecs-mapping.py`. +- From the repository root: `bash logstash/test-pipeline.sh`. +- Run Filebeat config checks with Podman, not Docker: + `podman run --rm --user 0 -e ELASTIC_PASSWORD=test -v "$PWD/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z" docker.elastic.co/beats/filebeat:9.3.3 filebeat test config -e -c /usr/share/filebeat/filebeat.yml` + +## Filebeat And Logs +- Filebeat configs should stay on `filestream`; use `parsers` for multiline and container parsing. +- Do not reintroduce `type: log` or `type: container` in Filebeat configs. +- `logs/` contains live sample inputs for Filebeat. +- `setup/generate-test-logs.py` writes directly to log files; its defaults point at `/var/log/...`, so override paths before using it on the host. + +## Scripts With Relative Paths +- `setup/import-rules.py` has brittle relative-path defaults; check `RULES_DIR` and `CACERT` before invoking it. +- Several helper scripts read `.env` from the current directory, so run them from the directory shown above. + +## Coding Approach +- Think before coding: state assumptions explicitly. If something is unclear or has multiple plausible interpretations, ask instead of guessing. +- Keep solutions minimal: implement only what was requested. Avoid speculative abstractions, configurability, or error handling for impossible scenarios. +- Make surgical changes: touch only what is necessary, match existing style, and do not refactor unrelated code or comments. Remove only unused imports, variables, or functions created by your own changes. +- Be goal-driven: for multi-step work, outline a brief plan and define how each step will be verified. + +## Content Rules +- Keep docs in Spanish and code/config identifiers in English. +- Do not use emojis under any circumstance. +- Preserve the 5 required detections, 3 attack simulations, and 2 response playbooks when editing project scope docs. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8ecba3c..6d30c23 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -5,7 +5,7 @@ applyTo: "**" # CloudSIEM Copilot Instructions ## Project scope -- This repository is an academic SIEM project built around `Elasticsearch`, `Logstash`, `Kibana`, `Filebeat` and/or `Metricbeat`, `Docker Compose`, Python log generators, and optional `Wazuh`. +- This repository is an academic SIEM project built around `Elasticsearch`, `Logstash`, `Kibana`, `Filebeat`, `Docker Compose` and Python log generators. - Follow the project scope, requirements, deliverables, and acceptance criteria defined in [`README.md`](../README.md). ## Source of truth @@ -14,10 +14,16 @@ applyTo: "**" ## Communication style - Use clear, direct, and concise language. -- Do not use emojis. +- Do not use emojis under any circumstance. - Prefer factual statements over promotional or decorative wording. - Keep explanations practical and focused on what exists, what is missing, and what should be implemented next. +## Coding approach +- Think before coding: state assumptions explicitly. If something is unclear or multiple interpretations exist, ask instead of guessing. +- Keep solutions minimal: implement only what was requested. Avoid speculative abstractions, configurability, or error handling for impossible scenarios. +- Make surgical changes: touch only what is necessary, match existing style, and do not refactor unrelated code or comments. Clean up only unused imports, variables, or functions created by your own changes. +- Be goal-driven: for multi-step work, outline a brief plan and define how each step will be verified. + ## Documentation rules - Write project documentation in Spanish. - Keep code, configuration keys, service names, index names, and technical identifiers in English. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e1daaae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,144 @@ +name: CI + +on: + pull_request: + branches: + - main + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install lint tools + run: | + python -m pip install --upgrade pip + python -m pip install ruff yamllint + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Lint Python + run: ruff check --select E9,F63,F7,F82 setup + + - name: Lint shell scripts + run: | + shellcheck -S error \ + logstash/test-pipeline.sh \ + setup/run-podman-sentinel-stack.sh \ + setup/provisioning/*.sh \ + setup/verify-cluster.sh + + - name: Lint YAML + run: | + yamllint -d '{ + extends: relaxed, + rules: { + colons: disable, + document-start: disable, + indentation: disable, + line-length: disable, + truthy: disable + } + }' \ + .github/workflows/ci.yml \ + docker-compose.yml \ + docker-compose-podman.yml \ + docker-compose-sentinel.yml \ + docker-compose-sentinel-podman.yml \ + filebeat/filebeat.yml \ + filebeat/filebeat-k8s.yml \ + filebeat/filebeat-daemonset.yaml \ + kibana/kibana.yml \ + logstash/config/logstash.yml \ + setup/provisioning/cert-instances.yml + + - name: Check line endings + run: | + bad=0 + while IFS=$'\t' read -r meta path; do + case "$meta" in + *"w/crlf"*|*"i/crlf"*) + printf 'CRLF detected: %s\t%s\n' "$meta" "$path" >&2 + bad=1 + ;; + esac + done < <(git ls-files --eol) + exit "$bad" + + - name: Validate JSON files + run: | + python - <<'PY' + import json + from pathlib import Path + + for path in sorted(Path('.').rglob('*.json')): + with path.open(encoding='utf-8') as handle: + json.load(handle) + print(path) + PY + + static-tests: + name: Static Tests + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Compile Python scripts + run: python -m compileall setup + + - name: Validate dashboard generation + run: | + python setup/create-dashboards-complete.py + git diff --exit-code setup/dashboards/executive-operational-dashboards.ndjson + + - name: Validate dashboard references + run: | + python - <<'PY' + import json + from pathlib import Path + + path = Path('setup/dashboards/executive-operational-dashboards.ndjson') + objects = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()] + ids = {(item['type'], item['id']) for item in objects} + missing = [] + + for item in objects: + if item.get('type') != 'dashboard': + continue + panels = json.loads(item['attributes']['panelsJSON']) + references = {ref['name']: (ref['type'], ref['id']) for ref in item.get('references', [])} + for panel in panels: + ref = references.get(panel.get('panelRefName')) + if ref not in ids: + missing.append((item['id'], panel.get('panelRefName'), ref)) + + if missing: + raise SystemExit(f'Missing dashboard references: {missing}') + print(f'Validated {len(objects)} saved objects') + PY + + - name: Validate Compose files + run: | + docker compose -f docker-compose.yml config --quiet + docker compose -f docker-compose.yml -f docker-compose-podman.yml config --quiet diff --git a/.gitignore b/.gitignore index e69de29..ad037a9 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,10 @@ +# Directorios y archivos a ignorar +setup/__pycache__/ +.ruff_cache/ + +# Ignorar variables de entorno +.env +.env.sentinel + +# Logs +logs \ No newline at end of file diff --git a/README.md b/README.md index a0b7156..4bc0621 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,66 @@ # CloudSIEM -## Plataforma SIEM simplificada para centralizar, normalizar, correlacionar y visualizar eventos de seguridad usando Elastic Stack. -Este repositorio corresponde al **Proyecto 8: Plataforma de análisis de logs y seguridad con Elastic Stack e inteligencia de amenazas** de la asignatura **Computación en la Nube**. +Guía del proyecto y del trabajo en `develop`. -## Descripción +## Resumen -El objetivo del proyecto es construir una plataforma de monitoreo y análisis de seguridad capaz de: - -- centralizar logs de múltiples fuentes heterogéneas; -- transformar los eventos a un esquema común; -- detectar comportamientos sospechosos mediante reglas y analítica; -- visualizar el estado de seguridad en dashboards ejecutivos y operacionales; -- demostrar la detección de ataques simulados en un entorno controlado. - -La solución está pensada como una implementación académica de un flujo SIEM end-to-end con componentes reales del ecosistema Elastic. +CloudSIEM es una plataforma SIEM académica basada en Elastic Stack para centralizar, normalizar, correlacionar y visualizar eventos de seguridad. El repositorio corresponde al Proyecto 8 de la asignatura Computación en la Nube. ## Objetivos - Centralizar y normalizar logs de múltiples fuentes. -- Implementar detección de anomalías y correlación de eventos. -- Construir dashboards de seguridad ejecutivos y operacionales. -- Simular escenarios de ataque y validar su detección. -- Documentar procedimientos de respuesta a incidentes. +- Detectar amenazas y patrones sospechosos. +- Visualizar el estado de seguridad en dashboards. +- Documentar respuesta a incidentes. +- Demostrar la detección de ataques simulados en un entorno controlado. ## Alcance del proyecto -La plataforma debe incluir, como mínimo: - -- `Elasticsearch` para almacenamiento, indexación e ILM. +- `Elasticsearch` para almacenamiento, indexación e `ILM`. - `Logstash` para parsing, transformación y enriquecimiento. - `Kibana` para observabilidad, seguridad y dashboards. -- `Filebeat` y/o `Metricbeat` para recolección de logs y métricas. -- `Docker Compose` o `Kubernetes` para despliegue. -- Scripts en `Python` para generar tráfico y eventos de prueba. -- `Wazuh` como componente opcional para detección adicional. +- `Filebeat` para recolección de logs. +- `Docker Compose` como despliegue principal; `Kubernetes` como ruta complementaria. +- `Python` para generadores de logs y utilidades. + +## Despliegue local con Podman en Linux + +Si se requiere desplegar el stack en local, solicitar al equipo de desarrollo las variables de entorno vigentes antes de ejecutar los comandos. Estas variables normalmente viven en `.env` y, si se habilita Sentinel, en `.env.sentinel`. No versionar credenciales ni secretos en el repositorio. + +El despliegue en Linux con Podman usa el launcher por etapas del repositorio: + +```bash +systemctl --user start podman.socket +bash setup/run-podman-sentinel-stack.sh --build --force-recreate --no-sentinel +``` + +Para incluir la integración opcional con Microsoft Sentinel: + +```bash +bash setup/run-podman-sentinel-stack.sh --build --force-recreate +``` + +Para detener el stack local: + +```bash +podman-compose -f docker-compose.yml -f docker-compose-podman.yml down +``` + +Para detener el stack con Sentinel: + +```bash +podman-compose -f docker-compose.yml -f docker-compose-podman.yml -f docker-compose-sentinel.yml -f docker-compose-sentinel-podman.yml down +``` + +El override `docker-compose-podman.yml` configura `userns_mode: keep-id` en Filebeat, relabeling SELinux para bind mounts y el socket rootless de Podman desde `$XDG_RUNTIME_DIR/podman/podman.sock`. + +El launcher ejecuta los jobs one-shot con el orden correcto y evita los fallos de Podman Compose con servicios one-shot (`setup`, `cleanup-indices`, `ilm-setup`, `data-views-setup` y `dashboards-import`) que terminan correctamente pero no quedan en estado `running`. En Sentinel, el relabeling SELinux queda en `docker-compose-sentinel-podman.yml` para que `docker-compose-sentinel.yml` siga siendo portable en Windows y Docker Desktop. + +Para Windows, Docker Desktop o WSL2 con Sentinel, usar: + +```bash +docker compose -f docker-compose.yml -f docker-compose-sentinel.yml up --build +``` ## Arquitectura propuesta @@ -45,8 +72,8 @@ Fuentes de logs |- Kubernetes `- Logs simulados por scripts -Beats / Recolección - `- Filebeat / Metricbeat +Filebeat / Recolección + `- Filebeat Pipeline de ingesta `- Logstash @@ -55,39 +82,47 @@ Pipeline de ingesta |- geoip `- routing de errores +Almacenamiento y gestión Almacenamiento y gestión `- Elasticsearch + |- índices ECS |- índices ECS |- ILM `- retención de 30 días + `- retención de 30 días +Analítica y visualización Analítica y visualización `- Kibana SIEM / Dashboards / Discover Detección |- reglas de correlación - `- machine learning / alertas + `- alertas ``` ## Fuentes de logs esperadas +El proyecto debe integrar al menos 3 fuentes de logs diferentes. Las fuentes objetivo definidas para la sustentación son: El proyecto debe integrar al menos 3 fuentes de logs diferentes. Las fuentes objetivo definidas para la sustentación son: - logs de sistema (`syslog`); - logs de seguridad (`auth.log`); - logs de aplicación web (`nginx` o `apache`); +- logs de aplicación web (`nginx` o `apache`); - logs de Kubernetes. Todos los eventos deben quedar visibles en Kibana y, en la medida de lo posible, alineados con `ECS` (Elastic Common Schema). +## Requerimientos funcionales y técnicos ## Requerimientos funcionales y técnicos +| ID | Requerimiento | Tecnologías principales | Criterio de aceptación | | ID | Requerimiento | Tecnologías principales | Criterio de aceptación | | --- | --- | --- | --- | -| `R8.1` | Desplegar `Elasticsearch` con 3 nodos, `Logstash` y `Kibana` con `Docker Compose`; configurar índices con `ILM` y retención de 30 días. | `Elasticsearch`, `Docker Compose` | Cluster en estado `green`; ILM aplicado y verificado; evidencia de paso a `warm` y/o `cold` en prueba acelerada. | -| `R8.2` | Configurar ingesta de logs desde mínimo 3 fuentes: sistema (`syslog`), aplicación web (`nginx` o `apache`), Kubernetes y/o seguridad (`auth.log`). | `Filebeat`, `Metricbeat`, `Logstash` | Logs visibles en Kibana; campos `ECS` correctamente mapeados. | +| `R8.1` | Desplegar `Elasticsearch` con 3 nodos, `Logstash` y `Kibana` con `Docker Compose`; configurar índices con `ILM` y retención de 30 días. | `Elasticsearch`, `Docker Compose` | Clúster en estado `green`; `ILM` aplicado y verificado; evidencia de paso a `warm` y/o `cold` en prueba acelerada. | +| `R8.2` | Configurar ingesta de logs desde mínimo 3 fuentes: sistema (`syslog`), aplicación web (`nginx` o `apache`), Kubernetes y/o seguridad (`auth.log`). | `Filebeat`, `Logstash` | Logs visibles en Kibana; campos `ECS` correctamente mapeados. | | `R8.3` | Implementar pipeline de `Logstash` con filtros `grok`, `mutate` y `geoip`; rechazar logs malformados a un índice de errores separado. | `Logstash`, `Grok`, `GeoIP` | Tasa de parseo exitoso mayor al 95%; logs rechazados almacenados con el motivo documentado. | -| `R8.4` | Implementar mínimo 5 reglas de detección: brute force SSH, escaneo de puertos, múltiples errores `404`, login fuera de horario y acceso a rutas sensibles. | `Kibana SIEM`, `Wazuh` opcional | Cada regla debe dispararse correctamente con logs de prueba generados para ese escenario. | +| `R8.4` | Implementar mínimo 5 reglas de detección: brute force SSH, escaneo de puertos, múltiples errores `404`, login fuera de horario y acceso a rutas sensibles. | `Kibana SIEM` | Cada regla debe dispararse correctamente con logs de prueba generados para ese escenario. | | `R8.5` | Simular mínimo 3 escenarios de ataque: brute force, escaneo con `nmap` e inyección SQL reflejada en logs. | `nmap`, `hydra`, `Python` | Los 3 escenarios deben ser detectados y generar alertas en menos de 60 segundos; incluir evidencias. | | `R8.6` | Construir dashboard ejecutivo con top amenazas del día, mapa geográfico de IPs sospechosas, tendencia de alertas por semana y salud general del sistema. | `Kibana Dashboards` | Dashboard con actualización automática y comprensible para audiencia no técnica. | | `R8.7` | Construir dashboard operacional con logs en tiempo real, alertas activas, top usuarios/IPs y drill-down a eventos específicos. | `Kibana Discover`, `Kibana Dashboards` | Investigación de incidente simulado completada en menos de 5 minutos usando el dashboard. | @@ -95,50 +130,67 @@ Todos los eventos deben quedar visibles en Kibana y, en la medida de lo posible, ## Entregables +- Stack ELK desplegado con mínimo 3 fuentes de logs diferentes. +- Mínimo 5 reglas de detección de amenazas configuradas y probadas. - Stack ELK desplegado con mínimo 3 fuentes de logs diferentes. - Mínimo 5 reglas de detección de amenazas configuradas y probadas. - Dashboard SIEM con vista ejecutiva y operacional. -- Playbook de respuesta para mínimo 2 tipos de incidente. +- Playbook de respuesta para mínimo 2 tipos de incidente en [`docs/playbooks/`](docs/playbooks/README.md). + +## Presentación + +Las diapositivas de la presentación también están disponibles en el repositorio, dentro de [`docs/Cloud-Native SIEM con Elastic Stack.pdf`](docs/Cloud-Native%20SIEM%20con%20Elastic%20Stack.pdf). ## Escenarios de ataque a demostrar +Los escenarios sugeridos para la validación del sistema son: Los escenarios sugeridos para la validación del sistema son: 1. `Brute force SSH` 2. `Escaneo de puertos con nmap` 3. `Inyección SQL registrada en logs web` +3. `Inyección SQL registrada en logs web` Para cada escenario se recomienda documentar: - fuente del log afectado; - patrón esperado en los eventos; +- patrón esperado en los eventos; - regla o alerta asociada; - evidencia en Kibana; - tiempo de detección; - acción de respuesta definida en el playbook. +## Integración opcional con Microsoft Sentinel + +El repositorio incluye una ruta cloud para detección continua con Microsoft Sentinel. La integración mantiene Elastic como SIEM local y agrega un forwarder que envía eventos normalizados de `logs-*` a una tabla `CloudSIEM_CL` en Log Analytics mediante Azure Monitor Logs Ingestion API. + +Los recursos Azure, reglas KQL y comandos CLI están documentados en [`azure/sentinel/README.md`](azure/sentinel/README.md). Esta ruta agrega reglas programadas de Sentinel para anomalías de autenticación SSH, patrones web sospechosos, escaneo de puertos, login fuera de horario y rutas sensibles. + ## Estructura esperada del repositorio +A medida que avance la implementación, este repositorio debería incorporar una estructura similar a la siguiente: A medida que avance la implementación, este repositorio debería incorporar una estructura similar a la siguiente: ```text . +├── .editorconfig +├── .gitattributes ├── README.md +├── .github/ +│ ├── AGENTS.md +│ └── copilot-instructions.md ├── docker-compose.yml -├── elasticsearch/ +├── docker-compose-podman.yml +├── docker-compose-sentinel.yml +├── docker-compose-sentinel-podman.yml +├── filebeat/ ├── logstash/ -│ ├── pipeline/ -│ └── config/ ├── kibana/ -│ ├── dashboards/ -│ └── alerts/ -├── beats/ -│ ├── filebeat/ -│ └── metricbeat/ -├── generators/ -│ └── python/ -├── playbooks/ -└── docs/ +├── setup/ +├── rules/ +├── docs/ +└── logs/ ``` ## Flujo de ramas @@ -166,17 +218,33 @@ Este repositorio usa `develop` como rama principal de integración del equipo. 4. Probar en `develop` antes de cualquier despliegue o demostración. 5. Abrir `Pull Request` preaprobado hacia `main` solo cuando el cambio esté listo para publicación o entrega. +## Antes de abrir PR a `main` + +Verificar que: + +- el cambio ya fue integrado en `develop`; +- las pruebas o validaciones necesarias ya se ejecutaron; +- la documentación relevante fue actualizada; +- el `Pull Request` a `main` tiene aprobación previa. + ## Criterios de éxito +Se considerará que el proyecto cumple su objetivo si logra: Se considerará que el proyecto cumple su objetivo si logra: - centralizar eventos de seguridad en una sola plataforma; - detectar ataques simulados con latencia baja; - ofrecer visibilidad técnica y ejecutiva; - facilitar investigación y respuesta mediante dashboards y playbooks. +- ofrecer visibilidad técnica y ejecutiva; +- facilitar investigación y respuesta mediante dashboards y playbooks. ## Estado del repositorio -Estado actual: `fase inicial de documentación`. +Estado actual: `implementación en curso`. + +Este README define el alcance, objetivos y entregables esperados del proyecto, junto con la guía operativa para trabajar en `develop`. + +## Nota -Este README define el alcance, objetivos y entregables esperados del proyecto. La implementación técnica del stack, pipelines, dashboards y playbooks debe incorporarse progresivamente en este repositorio. +La documentación del proyecto no debe usar emojis. diff --git a/azure/bicep/main.bicep b/azure/bicep/main.bicep new file mode 100644 index 0000000..b2597d3 --- /dev/null +++ b/azure/bicep/main.bicep @@ -0,0 +1,561 @@ +targetScope = 'resourceGroup' + +@description('Azure region for all Microsoft Sentinel resources.') +param location string = resourceGroup().location + +@description('Log Analytics workspace name.') +param workspaceName string = 'law-cloudsiem' + +@description('Data Collection Endpoint name used by the forwarder.') +param dataCollectionEndpointName string = 'dce-cloudsiem' + +@description('Data Collection Rule name used by the forwarder.') +param dataCollectionRuleName string = 'dcr-cloudsiem' + +@description('Object ID of the forwarder service principal. Leave empty to skip role assignment.') +param forwarderPrincipalObjectId string = '' + +@minValue(4) +@maxValue(730) +@description('Interactive retention for the custom CloudSIEM table.') +param retentionInDays int = 30 + +@description('Deploy CloudSIEM scheduled analytics rules in Microsoft Sentinel.') +param deployAnalyticsRules bool = true + +var customTableName = 'CloudSIEM_CL' +var streamName = 'Custom-CloudSIEM' +var workspaceDestinationName = 'cloudsiem-workspace' +var monitoringMetricsPublisherRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') + +var cloudSiemColumns = [ + { + name: 'TimeGenerated' + type: 'datetime' + } + { + name: 'EventId' + type: 'string' + } + { + name: 'ElasticIndex' + type: 'string' + } + { + name: 'EventDataset' + type: 'string' + } + { + name: 'EventCategory' + type: 'string' + } + { + name: 'EventType' + type: 'string' + } + { + name: 'EventAction' + type: 'string' + } + { + name: 'EventOutcome' + type: 'string' + } + { + name: 'EventSeverity' + type: 'long' + } + { + name: 'SiemSeverity' + type: 'string' + } + { + name: 'SourceIp' + type: 'string' + } + { + name: 'SourcePort' + type: 'long' + } + { + name: 'DestinationIp' + type: 'string' + } + { + name: 'DestinationPort' + type: 'long' + } + { + name: 'NetworkTransport' + type: 'string' + } + { + name: 'UserName' + type: 'string' + } + { + name: 'HostName' + type: 'string' + } + { + name: 'ProcessName' + type: 'string' + } + { + name: 'HttpMethod' + type: 'string' + } + { + name: 'HttpStatusCode' + type: 'long' + } + { + name: 'HttpResponseBytes' + type: 'long' + } + { + name: 'UrlPath' + type: 'string' + } + { + name: 'UrlOriginal' + type: 'string' + } + { + name: 'UserAgent' + type: 'string' + } + { + name: 'Tags' + type: 'dynamic' + } + { + name: 'BruteForceAttempt' + type: 'boolean' + } + { + name: 'AfterHoursLogin' + type: 'boolean' + } + { + name: 'SensitivePath' + type: 'boolean' + } + { + name: 'SqliDetected' + type: 'boolean' + } + { + name: 'XssDetected' + type: 'boolean' + } + { + name: 'PathTraversal' + type: 'boolean' + } + { + name: 'ScannerDetected' + type: 'boolean' + } + { + name: 'Message' + type: 'string' + } +] + +resource workspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: workspaceName + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: retentionInDays + features: { + searchVersion: 1 + enableLogAccessUsingOnlyResourcePermissions: true + } + } +} + +resource cloudSiemTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + parent: workspace + name: customTableName + properties: { + retentionInDays: retentionInDays + totalRetentionInDays: retentionInDays + schema: { + name: customTableName + columns: cloudSiemColumns + } + } +} + +resource sentinelOnboarding 'Microsoft.SecurityInsights/onboardingStates@2023-02-01-preview' = { + scope: workspace + name: 'default' + properties: {} +} + +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } +} + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: dataCollectionRuleName + location: location + kind: 'Direct' + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + streamDeclarations: { + 'Custom-CloudSIEM': { + columns: cloudSiemColumns + } + } + destinations: { + logAnalytics: [ + { + name: workspaceDestinationName + workspaceResourceId: workspace.id + } + ] + } + dataFlows: [ + { + streams: [ + streamName + ] + destinations: [ + workspaceDestinationName + ] + transformKql: 'source' + outputStream: 'Custom-${customTableName}' + } + ] + } + dependsOn: [ + cloudSiemTable + ] +} + +resource dcrPublisherRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (forwarderPrincipalObjectId != '') { + name: guid(dataCollectionRule.id, forwarderPrincipalObjectId, 'cloudsiem-monitoring-metrics-publisher') + scope: dataCollectionRule + properties: { + roleDefinitionId: monitoringMetricsPublisherRoleId + principalId: forwarderPrincipalObjectId + principalType: 'ServicePrincipal' + } +} + +resource authAnomalyRule 'Microsoft.SecurityInsights/alertRules@2023-02-01-preview' = if (deployAnalyticsRules) { + scope: workspace + name: guid(workspace.id, 'cloudsiem-auth-failed-login-anomaly') + kind: 'Scheduled' + properties: { + displayName: 'CloudSIEM - ML SSH Failed Login Spike' + description: 'Detecta picos anomalos de fallos de autenticacion SSH por IP origen usando series_decompose_anomalies en Microsoft Sentinel.' + severity: 'High' + enabled: true + query: loadTextContent('../sentinel/queries/auth-failed-login-anomaly.kql') + queryFrequency: 'PT5M' + queryPeriod: 'PT24H' + triggerOperator: 'GreaterThan' + triggerThreshold: 0 + suppressionDuration: 'PT1H' + suppressionEnabled: false + tactics: [ + 'CredentialAccess' + ] + entityMappings: [ + { + entityType: 'IP' + fieldMappings: [ + { + identifier: 'Address' + columnName: 'SourceIp' + } + ] + } + ] + customDetails: { + FailedLogins: 'FailedLogins' + AnomalyScore: 'AnomalyScore' + Baseline: 'Baseline' + } + incidentConfiguration: { + createIncident: true + groupingConfiguration: { + enabled: true + reopenClosedIncident: false + lookbackDuration: 'PT1H' + matchingMethod: 'Selected' + groupByEntities: [ + 'IP' + ] + groupByAlertDetails: [] + groupByCustomDetails: [] + } + } + eventGroupingSettings: { + aggregationKind: 'SingleAlert' + } + } + dependsOn: [ + sentinelOnboarding + ] +} + +resource webAnomalyRule 'Microsoft.SecurityInsights/alertRules@2023-02-01-preview' = if (deployAnalyticsRules) { + scope: workspace + name: guid(workspace.id, 'cloudsiem-web-suspicious-pattern-anomaly') + kind: 'Scheduled' + properties: { + displayName: 'CloudSIEM - ML Web Suspicious Pattern Spike' + description: 'Detecta picos anomalos de patrones web sospechosos: 404 masivos, scanners, SQLi, XSS, path traversal y rutas sensibles.' + severity: 'Medium' + enabled: true + query: loadTextContent('../sentinel/queries/web-suspicious-pattern-anomaly.kql') + queryFrequency: 'PT5M' + queryPeriod: 'PT24H' + triggerOperator: 'GreaterThan' + triggerThreshold: 0 + suppressionDuration: 'PT1H' + suppressionEnabled: false + tactics: [ + 'Reconnaissance' + 'InitialAccess' + ] + entityMappings: [ + { + entityType: 'IP' + fieldMappings: [ + { + identifier: 'Address' + columnName: 'SourceIp' + } + ] + } + ] + customDetails: { + SuspiciousRequests: 'SuspiciousRequests' + AnomalyScore: 'AnomalyScore' + Baseline: 'Baseline' + } + incidentConfiguration: { + createIncident: true + groupingConfiguration: { + enabled: true + reopenClosedIncident: false + lookbackDuration: 'PT1H' + matchingMethod: 'Selected' + groupByEntities: [ + 'IP' + ] + groupByAlertDetails: [] + groupByCustomDetails: [] + } + } + eventGroupingSettings: { + aggregationKind: 'SingleAlert' + } + } + dependsOn: [ + sentinelOnboarding + ] +} + +resource portscanAnomalyRule 'Microsoft.SecurityInsights/alertRules@2023-02-01-preview' = if (deployAnalyticsRules) { + scope: workspace + name: guid(workspace.id, 'cloudsiem-portscan-anomaly') + kind: 'Scheduled' + properties: { + displayName: 'CloudSIEM - ML Port Scan Spike' + description: 'Detecta picos anomalos de puertos destino distintos bloqueados por IP origen.' + severity: 'High' + enabled: true + query: loadTextContent('../sentinel/queries/portscan-anomaly.kql') + queryFrequency: 'PT5M' + queryPeriod: 'PT24H' + triggerOperator: 'GreaterThan' + triggerThreshold: 0 + suppressionDuration: 'PT1H' + suppressionEnabled: false + tactics: [ + 'Discovery' + 'Reconnaissance' + ] + entityMappings: [ + { + entityType: 'IP' + fieldMappings: [ + { + identifier: 'Address' + columnName: 'SourceIp' + } + ] + } + ] + customDetails: { + DistinctPorts: 'DistinctPorts' + AnomalyScore: 'AnomalyScore' + Baseline: 'Baseline' + } + incidentConfiguration: { + createIncident: true + groupingConfiguration: { + enabled: true + reopenClosedIncident: false + lookbackDuration: 'PT1H' + matchingMethod: 'Selected' + groupByEntities: [ + 'IP' + ] + groupByAlertDetails: [] + groupByCustomDetails: [] + } + } + eventGroupingSettings: { + aggregationKind: 'SingleAlert' + } + } + dependsOn: [ + sentinelOnboarding + ] +} + +resource afterHoursRule 'Microsoft.SecurityInsights/alertRules@2023-02-01-preview' = if (deployAnalyticsRules) { + scope: workspace + name: guid(workspace.id, 'cloudsiem-after-hours-login') + kind: 'Scheduled' + properties: { + displayName: 'CloudSIEM - Login SSH Fuera de Horario' + description: 'Detecta logins exitosos fuera del horario laboral America/Bogota o eventos marcados por Logstash como after_hours_login.' + severity: 'Medium' + enabled: true + query: loadTextContent('../sentinel/queries/after-hours-login.kql') + queryFrequency: 'PT5M' + queryPeriod: 'PT2H' + triggerOperator: 'GreaterThan' + triggerThreshold: 0 + suppressionDuration: 'PT1H' + suppressionEnabled: false + tactics: [ + 'InitialAccess' + 'Persistence' + ] + entityMappings: [ + { + entityType: 'IP' + fieldMappings: [ + { + identifier: 'Address' + columnName: 'SourceIp' + } + ] + } + { + entityType: 'Account' + fieldMappings: [ + { + identifier: 'Name' + columnName: 'UserName' + } + ] + } + ] + incidentConfiguration: { + createIncident: true + groupingConfiguration: { + enabled: true + reopenClosedIncident: false + lookbackDuration: 'PT2H' + matchingMethod: 'AllEntities' + groupByEntities: [] + groupByAlertDetails: [] + groupByCustomDetails: [] + } + } + eventGroupingSettings: { + aggregationKind: 'SingleAlert' + } + } + dependsOn: [ + sentinelOnboarding + ] +} + +resource sensitivePathRule 'Microsoft.SecurityInsights/alertRules@2023-02-01-preview' = if (deployAnalyticsRules) { + scope: workspace + name: guid(workspace.id, 'cloudsiem-sensitive-path-access') + kind: 'Scheduled' + properties: { + displayName: 'CloudSIEM - Acceso a Rutas Web Sensibles' + description: 'Detecta acceso a rutas web sensibles como admin, phpMyAdmin, .env, .git y endpoints internos.' + severity: 'High' + enabled: true + query: loadTextContent('../sentinel/queries/sensitive-path-access.kql') + queryFrequency: 'PT5M' + queryPeriod: 'PT2H' + triggerOperator: 'GreaterThan' + triggerThreshold: 0 + suppressionDuration: 'PT1H' + suppressionEnabled: false + tactics: [ + 'InitialAccess' + 'CredentialAccess' + ] + entityMappings: [ + { + entityType: 'IP' + fieldMappings: [ + { + identifier: 'Address' + columnName: 'SourceIp' + } + ] + } + { + entityType: 'URL' + fieldMappings: [ + { + identifier: 'Url' + columnName: 'UrlPath' + } + ] + } + ] + incidentConfiguration: { + createIncident: true + groupingConfiguration: { + enabled: true + reopenClosedIncident: false + lookbackDuration: 'PT2H' + matchingMethod: 'AllEntities' + groupByEntities: [] + groupByAlertDetails: [] + groupByCustomDetails: [] + } + } + eventGroupingSettings: { + aggregationKind: 'SingleAlert' + } + } + dependsOn: [ + sentinelOnboarding + ] +} + +output workspaceName string = workspace.name +output workspaceResourceId string = workspace.id +output customTableName string = customTableName +output dataCollectionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output dataCollectionRuleId string = dataCollectionRule.id +output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId +output streamName string = streamName diff --git a/azure/bicep/parameters.example.json b/azure/bicep/parameters.example.json new file mode 100644 index 0000000..cf2f085 --- /dev/null +++ b/azure/bicep/parameters.example.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "location": { + "value": "eastus" + }, + "workspaceName": { + "value": "law-cloudsiem" + }, + "dataCollectionEndpointName": { + "value": "dce-cloudsiem" + }, + "dataCollectionRuleName": { + "value": "dcr-cloudsiem" + }, + "forwarderPrincipalObjectId": { + "value": "" + }, + "retentionInDays": { + "value": 30 + }, + "deployAnalyticsRules": { + "value": true + } + } +} diff --git a/azure/sentinel/README.md b/azure/sentinel/README.md new file mode 100644 index 0000000..5e8a939 --- /dev/null +++ b/azure/sentinel/README.md @@ -0,0 +1,184 @@ +# Microsoft Sentinel para CloudSIEM + +Esta integración agrega una capa cloud de detección con Microsoft Sentinel. El stack local sigue usando Elastic para ingesta, parsing y dashboards; un forwarder continuo consulta `logs-*` en Elasticsearch y envía eventos normalizados a una tabla personalizada de Log Analytics llamada `CloudSIEM_CL`. + +## Región sugerida + +Usar `eastus`. Tiene buena disponibilidad para Log Analytics, Data Collection Rules, Data Collection Endpoints y Microsoft Sentinel en cuentas de estudiante. Si hay restricción de cuota, usar `eastus2`. + +## Comandos Azure CLI + +Instalar o actualizar Azure CLI y Bicep si hace falta: + +```bash +az version +az bicep upgrade +``` + +Iniciar sesión y seleccionar la suscripción Student: + +```bash +az login +az account list --output table +az account set --subscription "" +``` + +Registrar proveedores requeridos: + +```bash +az provider register --namespace Microsoft.OperationalInsights +az provider register --namespace Microsoft.OperationsManagement +az provider register --namespace Microsoft.Insights +az provider register --namespace Microsoft.SecurityInsights +az provider register --namespace Microsoft.Authorization +``` + +Crear el grupo de recursos: + +```bash +az group create \ + --name rg-cloudsiem-sentinel \ + --location eastus +``` + +Crear una app registration y service principal para el forwarder: + +```bash +FORWARDER_APP_NAME=sp-cloudsiem-sentinel-forwarder + +AZURE_CLIENT_ID=$(az ad app create \ + --display-name "$FORWARDER_APP_NAME" \ + --query appId \ + --output tsv) + +AZURE_CLIENT_SECRET=$(az ad app credential reset \ + --id "$AZURE_CLIENT_ID" \ + --display-name cloudsiem-forwarder \ + --years 1 \ + --query password \ + --output tsv) + +AZURE_TENANT_ID=$(az account show --query tenantId --output tsv) + +AZURE_OBJECT_ID=$(az ad sp create \ + --id "$AZURE_CLIENT_ID" \ + --query id \ + --output tsv) +``` + +Desplegar Sentinel, la tabla personalizada, DCR/DCE y las reglas: + +```bash +az deployment group create \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --template-file azure/bicep/main.bicep \ + --parameters azure/bicep/parameters.example.json \ + --parameters forwarderPrincipalObjectId="$AZURE_OBJECT_ID" +``` + +Obtener las salidas necesarias para el forwarder: + +```bash +SENTINEL_DCE_ENDPOINT=$(az deployment group show \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --query properties.outputs.dataCollectionEndpoint.value \ + --output tsv) + +SENTINEL_DCR_IMMUTABLE_ID=$(az deployment group show \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --query properties.outputs.dataCollectionRuleImmutableId.value \ + --output tsv) +``` + +Crear `.env.sentinel` local: + +```bash +cat > .env.sentinel <= 18 +| project TimeGenerated, SourceIp, UserName, HostName, EventAction, LocalTime, Message diff --git a/azure/sentinel/queries/auth-failed-login-anomaly.kql b/azure/sentinel/queries/auth-failed-login-anomaly.kql new file mode 100644 index 0000000..e4bb046 --- /dev/null +++ b/azure/sentinel/queries/auth-failed-login-anomaly.kql @@ -0,0 +1,10 @@ +CloudSIEM_CL +| where EventDataset == "system.auth" +| extend TagsText = tostring(Tags) +| where EventAction == "authentication_failure" or TagsText has "auth_failure" +| where isnotempty(SourceIp) +| make-series FailedLogins=count() default=0 on TimeGenerated from ago(24h) to now() step 5m by SourceIp +| extend (AnomalyFlag, AnomalyScore, Baseline) = series_decompose_anomalies(FailedLogins, 2.5, -1, "linefit") +| mv-expand TimeGenerated to typeof(datetime), FailedLogins to typeof(long), AnomalyFlag to typeof(int), AnomalyScore to typeof(double), Baseline to typeof(double) +| where AnomalyFlag == 1 and FailedLogins >= 5 +| project TimeGenerated, SourceIp, FailedLogins, Baseline, AnomalyScore diff --git a/azure/sentinel/queries/portscan-anomaly.kql b/azure/sentinel/queries/portscan-anomaly.kql new file mode 100644 index 0000000..9d61875 --- /dev/null +++ b/azure/sentinel/queries/portscan-anomaly.kql @@ -0,0 +1,10 @@ +CloudSIEM_CL +| where EventDataset == "system.syslog" +| extend TagsText = tostring(Tags) +| where EventAction == "firewall_block" or TagsText has "firewall_block" +| where isnotempty(SourceIp) and isnotempty(DestinationPort) +| make-series DistinctPorts=dcount(DestinationPort) default=0 on TimeGenerated from ago(24h) to now() step 5m by SourceIp +| extend (AnomalyFlag, AnomalyScore, Baseline) = series_decompose_anomalies(DistinctPorts, 2.5, -1, "linefit") +| mv-expand TimeGenerated to typeof(datetime), DistinctPorts to typeof(long), AnomalyFlag to typeof(int), AnomalyScore to typeof(double), Baseline to typeof(double) +| where AnomalyFlag == 1 and DistinctPorts >= 10 +| project TimeGenerated, SourceIp, DistinctPorts, Baseline, AnomalyScore diff --git a/azure/sentinel/queries/sensitive-path-access.kql b/azure/sentinel/queries/sensitive-path-access.kql new file mode 100644 index 0000000..1aaf68e --- /dev/null +++ b/azure/sentinel/queries/sensitive-path-access.kql @@ -0,0 +1,7 @@ +CloudSIEM_CL +| where EventDataset == "nginx.access" +| extend TagsText = tostring(Tags) +| where SensitivePath == true + or TagsText has "sensitive_path_access" + or UrlPath has_any ("/admin", "/wp-admin", "/phpmyadmin", "/.env", "/config", "/backup", "/.git", "/etc/passwd", "/proc/", "/server-status", "/actuator", "/api/admin") +| project TimeGenerated, SourceIp, UrlPath, HttpStatusCode, UserAgent, SensitivePath, Message diff --git a/azure/sentinel/queries/web-suspicious-pattern-anomaly.kql b/azure/sentinel/queries/web-suspicious-pattern-anomaly.kql new file mode 100644 index 0000000..0afbfe0 --- /dev/null +++ b/azure/sentinel/queries/web-suspicious-pattern-anomaly.kql @@ -0,0 +1,16 @@ +CloudSIEM_CL +| where EventDataset == "nginx.access" +| extend TagsText = tostring(Tags) +| where HttpStatusCode == 404 + or SensitivePath == true + or SqliDetected == true + or XssDetected == true + or PathTraversal == true + or ScannerDetected == true + or TagsText has_any ("http_404", "sensitive_path_access", "sqli_detected", "xss_detected", "path_traversal", "security_scanner") +| where isnotempty(SourceIp) +| make-series SuspiciousRequests=count() default=0 on TimeGenerated from ago(24h) to now() step 5m by SourceIp +| extend (AnomalyFlag, AnomalyScore, Baseline) = series_decompose_anomalies(SuspiciousRequests, 2.5, -1, "linefit") +| mv-expand TimeGenerated to typeof(datetime), SuspiciousRequests to typeof(long), AnomalyFlag to typeof(int), AnomalyScore to typeof(double), Baseline to typeof(double) +| where AnomalyFlag == 1 and SuspiciousRequests >= 10 +| project TimeGenerated, SourceIp, SuspiciousRequests, Baseline, AnomalyScore diff --git a/docker-compose-podman.yml b/docker-compose-podman.yml new file mode 100644 index 0000000..afb39bf --- /dev/null +++ b/docker-compose-podman.yml @@ -0,0 +1,64 @@ +x-podman: + in_pod: false + +services: + setup: + volumes: + - ./setup:/setup:ro,z + + es01: + volumes: + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro,z + + es02: + volumes: + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro,z + + es03: + volumes: + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro,z + + kibana: + volumes: + - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,z + - ./kibana/node.options:/usr/share/kibana/config/node.options:ro,z + + logstash: + volumes: + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,z + - ./logstash/config/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro,z + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,z + - ./logstash/patterns:/usr/share/logstash/patterns:ro,z + + filebeat: + userns_mode: keep-id + volumes: + - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,z + - ${XDG_RUNTIME_DIR:?XDG_RUNTIME_DIR must be set}/podman/podman.sock:/var/run/docker.sock:ro,z + - ./logs/:/var/log:ro,z + + cleanup-indices: + volumes: + - ./setup:/setup:ro,z + + ilm-setup: + volumes: + - ./setup:/setup:ro,z + + rules-import: + volumes: + - ./rules:/rules:rw,z + - ./setup:/setup:ro,z + + data-views-setup: + volumes: + - ./setup:/app/setup:ro,z + + dashboards-import: + volumes: + - ./setup:/app/setup:z + + log-generator: + volumes: + - ./setup:/app/setup:ro,z + - ./logs:/app/logs:z diff --git a/docker-compose-sentinel-podman.yml b/docker-compose-sentinel-podman.yml new file mode 100644 index 0000000..9806f63 --- /dev/null +++ b/docker-compose-sentinel-podman.yml @@ -0,0 +1,8 @@ +# ============================================================================= +# Podman-only Sentinel overlay +# ============================================================================= + +services: + sentinel-forwarder: + volumes: + - ./setup:/app/setup:ro,z diff --git a/docker-compose-sentinel.yml b/docker-compose-sentinel.yml new file mode 100644 index 0000000..8c37452 --- /dev/null +++ b/docker-compose-sentinel.yml @@ -0,0 +1,47 @@ +# ============================================================================= +# Optional Microsoft Sentinel streaming overlay +# Usage: +# Docker Compose / Windows / WSL2: +# docker compose -f docker-compose.yml -f docker-compose-sentinel.yml up --build +# Podman / Linux: +# bash setup/run-podman-sentinel-stack.sh --build --force-recreate +# (adds docker-compose-sentinel-podman.yml for SELinux relabeling) +# ============================================================================= + +volumes: + sentinel-forwarder-state: + driver: local + +services: + sentinel-forwarder: + image: python:3.11-slim + container_name: siem-sentinel-forwarder + depends_on: + es01: + condition: service_healthy + logstash: + condition: service_healthy + networks: + - elastic + volumes: + - certs:/certs:ro + - ./setup:/app/setup:ro + - sentinel-forwarder-state:/state + working_dir: /app + environment: + - ES_URL=https://es01:9200 + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - CACERT=/certs/ca/ca.crt + - AZURE_TENANT_ID=${AZURE_TENANT_ID:-} + - AZURE_CLIENT_ID=${AZURE_CLIENT_ID:-} + - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-} + - SENTINEL_DCE_ENDPOINT=${SENTINEL_DCE_ENDPOINT:-} + - SENTINEL_DCR_IMMUTABLE_ID=${SENTINEL_DCR_IMMUTABLE_ID:-} + - SENTINEL_STREAM_NAME=${SENTINEL_STREAM_NAME:-Custom-CloudSIEM} + - SENTINEL_POLL_SECONDS=${SENTINEL_POLL_SECONDS:-10} + - SENTINEL_BATCH_SIZE=${SENTINEL_BATCH_SIZE:-500} + - SENTINEL_LOOKBACK_MINUTES=${SENTINEL_LOOKBACK_MINUTES:-15} + - SENTINEL_CHECKPOINT_FILE=/state/checkpoint.json + entrypoint: ["python", "setup/sentinel-forwarder.py"] + restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c51fea0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,433 @@ +# ============================================================================= +# SIEM Platform — Elastic Stack (ELK) — 3-Node Cluster +# Proyecto 8: Plataforma de análisis de logs y seguridad +# VERSIÓN UNIVERSAL +# ============================================================================= +# Uso: +# 1. Crear archivo .env con las variables requeridas +# 2. En Linux con Podman: `bash setup/run-podman-sentinel-stack.sh --build --force-recreate` +# Agrega `--no-sentinel` para arrancar solo el stack local. +# 3. Esperar ~90s y verificar: bash setup/verify-cluster.sh +# ============================================================================= + +# ─── Volumes ───────────────────────────────────────────────────────────────── +volumes: + es01-data: + driver: local + es02-data: + driver: local + es03-data: + driver: local + kibana-data: + driver: local + logstash-data: + driver: local + certs: + driver: local + +# ─── Networks ──────────────────────────────────────────────────────────────── +networks: + elastic: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/24 + +# ─── Services ──────────────────────────────────────────────────────────────── +services: + # ─── Setup: genera certificados TLS y bootstrap del cluster ─────────────── + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: siem-setup + volumes: + - certs:/usr/share/elasticsearch/config/certs + - ./setup:/setup:ro + user: "0" + networks: + - elastic + command: ["bash", "/setup/provisioning/bootstrap-certs.sh"] + + # ─── Elasticsearch Nodo 1 (master eligible + data) ──────────────────────── + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: siem-es01 + depends_on: + setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.10 + ports: + - "9200:9200" + volumes: + - certs:/usr/share/elasticsearch/config/certs + - es01-data:/usr/share/elasticsearch/data + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es02,es03 + - node.roles=master,data,ingest,transform,remote_cluster_client + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=basic + - xpack.ml.enabled=true + - ES_JAVA_OPTS=-Xms512m -Xmx512m + healthcheck: + test: [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | + grep -q 'missing authentication'", + ] + interval: 10s + timeout: 10s + retries: 120 + + # ─── Elasticsearch Nodo 2 (master eligible + data) ──────────────────────── + es02: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: siem-es02 + depends_on: + setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.11 + volumes: + - certs:/usr/share/elasticsearch/config/certs + - es02-data:/usr/share/elasticsearch/data + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + environment: + - node.name=es02 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es03 + - node.roles=master,data,ingest,transform,remote_cluster_client + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es02/es02.key + - xpack.security.http.ssl.certificate=certs/es02/es02.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es02/es02.key + - xpack.security.transport.ssl.certificate=certs/es02/es02.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=basic + - xpack.ml.enabled=true + - ES_JAVA_OPTS=-Xms512m -Xmx512m + healthcheck: + test: [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | + grep -q 'missing authentication'", + ] + interval: 10s + timeout: 10s + retries: 120 + + # ─── Elasticsearch Nodo 3 (master eligible + data) ──────────────────────── + es03: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: siem-es03 + depends_on: + setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.12 + volumes: + - certs:/usr/share/elasticsearch/config/certs + - es03-data:/usr/share/elasticsearch/data + - ./elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + environment: + - node.name=es03 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es02 + - node.roles=master,data,ingest,transform,remote_cluster_client + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es03/es03.key + - xpack.security.http.ssl.certificate=certs/es03/es03.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es03/es03.key + - xpack.security.transport.ssl.certificate=certs/es03/es03.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=basic + - xpack.ml.enabled=true + - ES_JAVA_OPTS=-Xms512m -Xmx512m + healthcheck: + test: [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | + grep -q 'missing authentication'", + ] + interval: 10s + timeout: 10s + retries: 120 + + # ─── Kibana ─────────────────────────────────────────────────────────────── + kibana: + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + container_name: siem-kibana + depends_on: + es01: + condition: service_healthy + es02: + condition: service_healthy + es03: + condition: service_healthy + ilm-setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.20 + ports: + - "5601:5601" + volumes: + - certs:/usr/share/kibana/config/certs + - kibana-data:/usr/share/kibana/data + - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml:ro + - ./kibana/node.options:/usr/share/kibana/config/node.options:ro + environment: + - SERVER_NAME=${SERVER_NAME} + - 'ELASTICSEARCH_HOSTS=["https://es01:9200","https://es02:9200","https://es03:9200"]' + - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - XPACK_SECURITY_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + - XPACK_REPORTING_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -o /dev/null -w '%{http_code}' --cacert /usr/share/kibana/config/certs/ca/ca.crt https://localhost:5601 | grep -q '^302$'", + ] + interval: 20s + timeout: 10s + retries: 120 + + # ─── Logstash ───────────────────────────────────────────────────────────── + logstash: + image: docker.elastic.co/logstash/logstash:${STACK_VERSION} + container_name: siem-logstash + depends_on: + es01: + condition: service_healthy + kibana: + condition: service_healthy + ilm-setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.30 + ports: + - "5044:5044" + - "5514:5514/udp" + - "5515:5514/tcp" + - "9600:9600" + volumes: + - certs:/usr/share/logstash/config/certs + - logstash-data:/usr/share/logstash/data + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - ./logstash/config/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro + - ./logstash/patterns:/usr/share/logstash/patterns:ro + environment: + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - ELASTIC_HOSTS=https://es01:9200 + - LS_JAVA_OPTS=-Xms512m -Xmx512m + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9600 | grep -q 'green'"] + interval: 10s + timeout: 10s + retries: 60 + + # ─── Filebeat ───────────────────────────────────────────────────────────── + filebeat: + image: docker.elastic.co/beats/filebeat:${STACK_VERSION} + container_name: siem-filebeat + user: root + depends_on: + es01: + condition: service_healthy + kibana: + condition: service_healthy + logstash: + condition: service_healthy + networks: + - elastic + volumes: + - certs:/usr/share/filebeat/certs:ro + - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ${HOST_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock:ro + - ./logs/:/var/log:ro + environment: + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - ELASTIC_HOSTS=https://es01:9200 + - LOGSTASH_HOST=logstash:5044 + - KIBANA_HOST=https://kibana:5601 + command: ["filebeat", "-e", "--strict.perms=false"] + cap_add: + - DAC_READ_SEARCH + + # ─── Cleanup de indices SIEM (corre antes de ilm-setup) ───────────────────── + cleanup-indices: + image: python:3.11-slim + container_name: siem-cleanup-indices + user: "0" + depends_on: + es01: + condition: service_healthy + networks: + - elastic + volumes: + - certs:/certs:ro + - ./setup:/setup:ro + working_dir: /setup + environment: + - ELASTIC_HOSTS=https://es01:9200 + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - CACERT=/certs/ca/ca.crt + entrypoint: ["sh", "/setup/provisioning/cleanup-indices.sh"] + restart: "no" + + # ─── ILM Setup Job (corre una vez al inicio) ────────────────────────────── + ilm-setup: + image: curlimages/curl:latest + container_name: siem-ilm-setup + user: "0" + depends_on: + es01: + condition: service_healthy + es02: + condition: service_healthy + es03: + condition: service_healthy + cleanup-indices: + condition: service_completed_successfully + networks: + - elastic + volumes: + - certs:/certs + - ./setup:/setup:ro + environment: + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - KIBANA_PASSWORD=${KIBANA_PASSWORD} + entrypoint: ["sh", "/setup/provisioning/ilm-setup.sh"] + restart: "no" + + # ─── Importación automática de reglas Kibana ─────────────────────────────── + rules-import: + image: python:3.11-alpine + container_name: siem-rules-import + user: "0" + depends_on: + kibana: + condition: service_healthy + logstash: + condition: service_healthy + networks: + elastic: + ipv4_address: 172.20.0.40 + volumes: + - certs:/certs:ro + - ./rules:/rules:rw + - ./setup:/setup:ro + working_dir: /rules + environment: + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - KIBANA_HOST=https://kibana:5601 + - CACERT=/certs/ca/ca.crt + entrypoint: ["sh", "/setup/provisioning/import-rules.sh"] + restart: "no" + + # ─── Creación de Data Views en Kibana ─────────────────────────────────── + data-views-setup: + image: python:3.11-slim + container_name: siem-data-views-setup + user: "0" + depends_on: + kibana: + condition: service_healthy + networks: + elastic: + ipv4_address: 172.20.0.42 + volumes: + - certs:/certs:ro + - ./setup:/app/setup:ro + working_dir: /app + environment: + - KIBANA_HOST=https://kibana:5601 + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - CACERT=/certs/ca/ca.crt + - LOGS_DATA_VIEW_ID=logs-* + - LOGS_DATA_VIEW_TITLE=logs-* + entrypoint: ["sh", "/app/setup/provisioning/create-data-views.sh"] + restart: "no" + + # ─── Importación automática de dashboards Kibana ────────────────────────── + dashboards-import: + image: python:3.11-slim + container_name: siem-dashboards-import + user: "0" + depends_on: + kibana: + condition: service_healthy + data-views-setup: + condition: service_completed_successfully + networks: + elastic: + ipv4_address: 172.20.0.41 + volumes: + - certs:/certs:ro + - ./setup:/app/setup + working_dir: /app + environment: + - KIBANA_HOST=https://kibana:5601 + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - CACERT=/certs/ca/ca.crt + - LOGS_DATA_VIEW_ID=logs-* + - LOGS_DATA_VIEW_TITLE=logs-* + - INCLUDE_INDEX_PATTERNS=false + entrypoint: ["sh", "/app/setup/provisioning/import-dashboards.sh"] + restart: "no" + + # ─── Generación automática de logs de prueba ─────────────────────────── + log-generator: + image: python:3.11-slim + container_name: siem-log-generator + user: "0" + depends_on: + logstash: + condition: service_healthy + networks: + - elastic + volumes: + - ./setup:/app/setup:ro + - ./logs:/app/logs + working_dir: /app + entrypoint: ["sh", "/app/setup/provisioning/generate-logs.sh"] + restart: "no" diff --git a/docs/Cloud-Native SIEM con Elastic Stack.pdf b/docs/Cloud-Native SIEM con Elastic Stack.pdf new file mode 100644 index 0000000..245b567 Binary files /dev/null and b/docs/Cloud-Native SIEM con Elastic Stack.pdf differ diff --git a/docs/azure-sentinel.md b/docs/azure-sentinel.md new file mode 100644 index 0000000..aa3da9c --- /dev/null +++ b/docs/azure-sentinel.md @@ -0,0 +1,190 @@ +# Microsoft Sentinel para CloudSIEM + +Esta integración agrega una capa cloud de detección con Microsoft Sentinel. El stack local sigue usando Elastic para ingesta, parsing y dashboards; un forwarder continuo consulta `logs-*` en Elasticsearch y envía eventos normalizados a una tabla personalizada de Log Analytics llamada `CloudSIEM_CL`. + +## Región sugerida + +Usar `eastus`. Tiene buena disponibilidad para Log Analytics, Data Collection Rules, Data Collection Endpoints y Microsoft Sentinel en cuentas de estudiante. Si hay restricción de cuota, usar `eastus2`. + +## Recursos creados + +- Resource group: `rg-cloudsiem-sentinel` +- Log Analytics Workspace: `law-cloudsiem` +- Microsoft Sentinel onboarding sobre el workspace +- Tabla personalizada: `CloudSIEM_CL` +- Data Collection Endpoint: `dce-cloudsiem` +- Data Collection Rule: `dcr-cloudsiem` +- 5 reglas programadas de Sentinel para patrones sospechosos + +## Comandos Azure CLI + +Instalar o actualizar Azure CLI y Bicep si hace falta: + +```bash +az version +az bicep upgrade +``` + +Iniciar sesión y seleccionar la suscripción Student: + +```bash +az login +az account list --output table +az account set --subscription "" +``` + +Registrar proveedores requeridos: + +```bash +az provider register --namespace Microsoft.OperationalInsights +az provider register --namespace Microsoft.Insights +az provider register --namespace Microsoft.SecurityInsights +az provider register --namespace Microsoft.Authorization +``` + +Crear el grupo de recursos: + +```bash +az group create \ + --name rg-cloudsiem-sentinel \ + --location eastus +``` + +Crear una app registration y service principal para el forwarder: + +```bash +FORWARDER_APP_NAME=sp-cloudsiem-sentinel-forwarder + +AZURE_CLIENT_ID=$(az ad app create \ + --display-name "$FORWARDER_APP_NAME" \ + --query appId \ + --output tsv) + +AZURE_CLIENT_SECRET=$(az ad app credential reset \ + --id "$AZURE_CLIENT_ID" \ + --display-name cloudsiem-forwarder \ + --years 1 \ + --query password \ + --output tsv) + +AZURE_TENANT_ID=$(az account show --query tenantId --output tsv) + +AZURE_OBJECT_ID=$(az ad sp create \ + --id "$AZURE_CLIENT_ID" \ + --query id \ + --output tsv) +``` + +Desplegar Sentinel, la tabla personalizada, DCR/DCE y las reglas: + +```bash +az deployment group create \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --template-file azure/bicep/main.bicep \ + --parameters azure/bicep/parameters.example.json \ + --parameters forwarderPrincipalObjectId="$AZURE_OBJECT_ID" +``` + +Obtener las salidas necesarias para el forwarder: + +```bash +SENTINEL_DCE_ENDPOINT=$(az deployment group show \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --query properties.outputs.dataCollectionEndpoint.value \ + --output tsv) + +SENTINEL_DCR_IMMUTABLE_ID=$(az deployment group show \ + --name cloudsiem-sentinel \ + --resource-group rg-cloudsiem-sentinel \ + --query properties.outputs.dataCollectionRuleImmutableId.value \ + --output tsv) +``` + +Crear `.env.sentinel` local: + +```bash +cp .env.sentinel.example .env.sentinel + +cat > .env.sentinel < Discover que los índices `logs-syslog-*`, `logs-auth-*`, +`logs-nginx-*` y `logs-k8s-*` tienen documentos. + +### 2. Verificar tasa de parseo (R8.3) + +```bash +python setup/verify-parse-rate.py +``` + +Si ejecutas el verificador desde el host y no existe `./setup/certs/ca/ca.crt`, +el script desactiva la verificación TLS automáticamente. Si prefieres forzarla, +pasa `--cacert ` o `--insecure`. + +Debe reportar > 95%. + +### 3. Generar ataques y verificar detección (R8.4, R8.5) + +```bash +# Opción A: orquestador modo ataques +python setup/orchestrate-logs.py --mode attacks + +# Opción B: script individual +python setup/generate-test-logs.py --attack all_attacks +``` + +Esperar ~30-60s y verificar en Kibana -> Security -> Alerts: + +- `SIEM - Brute Force SSH Detectado` +- `SIEM - Escaneo de Puertos Detectado` +- `SIEM - Flood de Errores HTTP 404` +- `SIEM - Login SSH Exitoso Fuera de Horario Laboral` +- `SIEM - Acceso a Rutas Web Sensibles` + +### 4. Demostración completa (mix) + +```bash +python setup/orchestrate-logs.py --mode random +``` + +Esto genera una mezcla realista de tráfico normal y ataques, simulando un +entorno de producción real. + +--- + +## Solución de problemas + +**Los logs no aparecen en Kibana:** +```bash +# Verificar que Filebeat está corriendo +docker ps | grep filebeat + +# Verificar que los archivos existen +ls -la logs/ + +# Verificar conexión Logstash +curl -s http://localhost:9600/ | python3 -m json.tool +``` + +**La tasa de parseo es baja:** +```bash +# Revisar logs de Logstash +docker logs siem-logstash --tail 50 + +# Verificar contenido del índice de errores +curl -sk https://localhost:9200/logs-errors-*/_search?pretty \ + -u elastic:${ELASTIC_PASSWORD} +``` diff --git a/docs/playbooks/README.md b/docs/playbooks/README.md new file mode 100644 index 0000000..5553e5b --- /dev/null +++ b/docs/playbooks/README.md @@ -0,0 +1,88 @@ +# Guía operativa de playbooks de respuesta + +Esta carpeta contiene los playbooks de respuesta a incidentes del proyecto CloudSIEM. Su objetivo es que una persona que no participó en la creación del playbook pueda investigar, contener, erradicar y documentar un incidente simulado usando Kibana y los logs generados por el laboratorio. + +## Alcance + +| Escenario | Playbook | Alerta o indicador principal | Fuente | +| --- | --- | --- | --- | +| Fuerza bruta SSH | [brute-force-response.md](brute-force-response.md) | `SIEM - Brute Force SSH Detectado` | `logs-auth*` | +| SQLi y rutas web sensibles | [sqli-response.md](sqli-response.md) | `SIEM - Acceso a Rutas Web Sensibles`, `siem.sqli_detected:true` | `logs-nginx*` | + +## Requisitos previos + +Antes de ejecutar cualquier playbook, verificar que: + +1. El stack ELK esté levantado y Kibana sea accesible. +2. Las reglas de detección estén importadas y habilitadas. +3. Filebeat esté enviando eventos hacia Logstash y Elasticsearch. +4. El operador tenga acceso a `Kibana -> Security -> Alerts`, `Discover` y al dashboard `Operational - Consola de Triaje`, si fue importado. +5. La hora de Kibana cubra al menos los últimos 15 minutos durante simulaciones. + +## Flujo común de respuesta + +Cada playbook sigue el mismo flujo operativo: + +1. Identificar la alerta o indicador. +2. Reunir evidencia mínima: hora, regla, severidad, `source.ip`, host afectado, usuario o URL, cantidad de eventos y resultado de la petición o autenticación. +3. Investigar eventos relacionados en la misma ventana temporal. +4. Clasificar el incidente como falso positivo, intento bloqueado o compromiso probable. +5. Ejecutar contención según el nivel de impacto. +6. Ejecutar erradicación y corrección de causa raíz. +7. Registrar lecciones aprendidas y mejoras de detección. + +## Evidencia mínima por incidente + +| Campo | Por qué importa | +| --- | --- | +| `@timestamp` | Define la ventana de investigación. | +| `kibana.alert.rule.name` | Confirma que se disparó la regla esperada. | +| `event.dataset` | Identifica la fuente de logs. | +| `event.action` | Resume la acción observada. | +| `event.outcome` | Distingue intentos fallidos de accesos exitosos. | +| `source.ip` | Identifica el origen del ataque o simulación. | +| `host.name` | Identifica el activo afectado. | +| `user.name` | Aplica a autenticación SSH. | +| `url.path`, `url.query`, `url.original` | Aplica a ataques web. | +| `http.response.status_code` | Indica si el intento web fue bloqueado o exitoso. | + +## Criterio de aceptación del playbook + +El playbook se considera útil si un integrante no involucrado en su creación puede: + +1. Generar o recibir el incidente simulado. +2. Encontrar la alerta o indicador correcto en Kibana. +3. Completar la sección de investigación con evidencia concreta. +4. Tomar una decisión de contención justificada. +5. Ejecutar o describir la acción de erradicación adecuada. +6. Registrar al menos tres lecciones aprendidas o mejoras. +7. Cerrar el incidente con una conclusión reproducible. + +## Cómo simular los escenarios + +Desde la raíz del repositorio: + +```bash +python3 setup/generate-test-logs.py --attack brute_force --count 35 +python3 setup/generate-test-logs.py --attack sqli +``` + +Después de generar los logs, esperar de 30 a 60 segundos y revisar `Kibana -> Security -> Alerts` o `Discover` con los filtros indicados en cada playbook. + +## Formato de cierre + +Registrar el cierre con este resumen: + +| Dato | Valor | +| --- | --- | +| Playbook usado | | +| Fecha y hora de inicio | | +| Fecha y hora de cierre | | +| Alerta o indicador | | +| IP origen | | +| Activo afectado | | +| Impacto confirmado | | +| Contención aplicada | | +| Erradicación aplicada | | +| Evidencia guardada | | +| Lecciones aprendidas | | diff --git a/docs/playbooks/brute-force-response.md b/docs/playbooks/brute-force-response.md new file mode 100644 index 0000000..e6f8eb4 --- /dev/null +++ b/docs/playbooks/brute-force-response.md @@ -0,0 +1,224 @@ +# Playbook de respuesta: fuerza bruta SSH + +## Objetivo + +Guiar la respuesta ante múltiples intentos fallidos de autenticación SSH desde una misma IP. El playbook cubre investigación, contención, erradicación y lecciones aprendidas para un incidente simulado en CloudSIEM. + +## Resumen del escenario + +| Dato | Valor | +| --- | --- | +| Alerta principal | `SIEM - Brute Force SSH Detectado` | +| Severidad esperada | `high` | +| Regla | `rules/brute-force-ssh.json` | +| Fuente | `logs-auth*` | +| Dataset | `event.dataset:"system.auth"` | +| Condición | 5 o más fallos de autenticación desde la misma `source.ip` en 60 segundos | +| Técnica MITRE ATT&CK | `T1110.001 - Password Guessing` | +| Script de simulación | `python3 setup/generate-test-logs.py --attack brute_force --count 35` | + +## Criterios de activación + +Ejecutar este playbook si se cumple al menos una condición: + +1. Se genera una alerta `SIEM - Brute Force SSH Detectado` en `Kibana -> Security -> Alerts`. +2. En `Discover` se observan múltiples eventos `event.action:"authentication_failure"` desde la misma `source.ip`. +3. El dashboard operacional muestra un aumento anormal de fallos SSH. + +## Roles mínimos + +| Rol | Responsabilidad | +| --- | --- | +| Operador SIEM | Ejecuta la investigación y registra evidencia. | +| Responsable de infraestructura | Aplica bloqueo de IP, firewall o cambios de SSH si corresponde. | +| Responsable de identidad | Restablece credenciales o deshabilita usuarios si hubo éxito de autenticación. | + +## Preparación + +1. Abrir `Kibana -> Security -> Alerts`. +2. Seleccionar una ventana de tiempo de los últimos 15 minutos. +3. Confirmar que el índice `logs-auth*` contiene eventos recientes. +4. Si es una simulación, generar el escenario desde la raíz del repositorio: + +```bash +python3 setup/generate-test-logs.py --attack brute_force --count 35 +``` + +Esperar de 30 a 60 segundos para que Filebeat, Logstash y Elasticsearch procesen los eventos. + +## Fase 1: identificación + +1. Buscar la alerta `SIEM - Brute Force SSH Detectado`. +2. Registrar estos datos de la alerta: + +| Campo | Valor a registrar | +| --- | --- | +| Hora de alerta | `@timestamp` o `kibana.alert.start` | +| Regla | `kibana.alert.rule.name` | +| Severidad | `kibana.alert.severity` | +| IP origen | `source.ip` | +| Host afectado | `host.name` | +| Usuario objetivo | `user.name` | +| Acción | `event.action` | +| Resultado | `event.outcome` | + +3. Si no aparece la alerta, abrir `Discover` y usar esta consulta KQL: + +```text +event.dataset:"system.auth" and event.action:"authentication_failure" and process.name:"sshd" +``` + +4. Confirmar que una misma `source.ip` genera 5 o más fallos en 60 segundos. + +## Fase 2: investigación + +### Paso 1: determinar origen y volumen + +En `Discover`, filtrar por la IP origen de la alerta: + +```text +event.dataset:"system.auth" and source.ip:"" +``` + +Registrar: + +| Pregunta | Evidencia esperada | +| --- | --- | +| Cuántos intentos hubo | Conteo de eventos en la ventana temporal. | +| Desde qué IP vinieron | `source.ip`. | +| Contra qué host | `host.name`. | +| Contra qué usuarios | Lista de `user.name`. | +| Eran usuarios válidos o genéricos | `root`, `admin`, `ubuntu`, `postgres`, `test`, otros. | + +### Paso 2: buscar autenticación exitosa posterior + +Usar la misma IP origen y ampliar la ventana a 30 minutos: + +```text +event.dataset:"system.auth" and source.ip:"" and event.action:"authentication_success" +``` + +Interpretación: + +| Resultado | Clasificación | +| --- | --- | +| No hay eventos exitosos | Intento de brute force bloqueado o fallido. | +| Hay `authentication_success` después de los fallos | Compromiso probable de cuenta. | +| Hay éxito desde IP interna conocida y usuario validado | Posible falso positivo, requiere confirmación. | + +### Paso 3: revisar actividad posterior del usuario + +Si hubo autenticación exitosa, buscar actividad de sesión y comandos con privilegios: + +```text +event.dataset:"system.auth" and user.name:"" and (event.action:"session_opened" or event.action:"sudo_command" or tags:"privilege_escalation") +``` + +Registrar cualquier `process.command_line`, `host.name`, `user.name` y `@timestamp` observado. + +### Paso 4: descartar falsos positivos + +Considerar falso positivo solo si se puede demostrar una de estas condiciones: + +| Condición | Evidencia requerida | +| --- | --- | +| Usuario legítimo olvidó su clave | Confirmación del usuario y origen esperado. | +| Servicio interno tiene credencial vencida | IP interna documentada y owner identificado. | +| Prueba autorizada | Ventana de prueba aprobada y origen conocido. | + +Si no existe evidencia suficiente, tratar el evento como incidente real o simulación maliciosa confirmada. + +## Fase 3: contención + +### Acciones para laboratorio + +1. Marcar la alerta como `acknowledged` o `in progress` en Kibana si la opción está disponible. +2. Documentar la IP atacante y el usuario objetivo. +3. No modificar servicios productivos durante la simulación. +4. Si se requiere demostrar contención, registrar el comando que se aplicaría: + +```bash +sudo ufw deny from to any port 22 +``` + +### Acciones para entorno real + +Aplicar según el resultado de investigación: + +| Condición | Acción de contención | +| --- | --- | +| Solo fallos, sin éxito | Bloquear `source.ip` en firewall o security group. | +| Éxito posterior desde la misma IP | Bloquear IP, revocar sesiones y deshabilitar usuario afectado. | +| Múltiples usuarios atacados | Bloquear IP y revisar políticas de lockout. | +| Usuario privilegiado afectado | Escalar a incidente crítico y revocar credenciales de inmediato. | + +Comandos de referencia para Linux: + +```bash +sudo ufw deny from to any port 22 +sudo pkill -u +sudo passwd -l +``` + +Usar estos comandos solo con aprobación del responsable del sistema. + +## Fase 4: erradicación + +1. Forzar cambio de contraseña del usuario afectado si hubo éxito de autenticación. +2. Revocar llaves SSH no reconocidas en `~/.ssh/authorized_keys`. +3. Revisar cuentas con permisos elevados y accesos recientes. +4. Habilitar MFA donde aplique. +5. Validar que `PasswordAuthentication` esté deshabilitado si la política exige solo llaves SSH. +6. Ajustar `MaxAuthTries`, `AllowUsers` o controles equivalentes en `sshd_config`. +7. Confirmar que no vuelven a aparecer eventos desde la misma `source.ip` después de la contención. + +Consulta de verificación: + +```text +event.dataset:"system.auth" and source.ip:"" and @timestamp >= "" +``` + +## Fase 5: recuperación y monitoreo + +1. Mantener monitoreo por 30 minutos después de la contención. +2. Verificar que no existan nuevos `authentication_success` sospechosos. +3. Validar que los usuarios legítimos puedan autenticarse con el método permitido. +4. Cerrar la alerta solo cuando el origen esté bloqueado o clasificado como benigno. + +## Lecciones aprendidas + +Completar estas preguntas al cierre: + +| Pregunta | Respuesta | +| --- | --- | +| La detección se generó en menos de 60 segundos | | +| La alerta mostró la IP origen correctamente | | +| Hubo autenticación exitosa posterior | | +| La política de bloqueo de cuentas fue suficiente | | +| El firewall bloqueó automáticamente o requirió acción manual | | +| Se requiere MFA, allowlist o deshabilitar contraseñas SSH | | +| Qué ajuste se hará a reglas, dashboards o documentación | | + +## Checklist de cierre + +| Ítem | Estado | +| --- | --- | +| Alerta identificada | | +| IP origen registrada | | +| Usuarios objetivo registrados | | +| Se buscó autenticación exitosa posterior | | +| Se clasificó el incidente | | +| Se definió contención | | +| Se definió erradicación | | +| Se documentaron lecciones aprendidas | | +| Otro integrante pudo seguir el procedimiento | | + +## Resultado esperado en simulación + +Para considerar exitosa la prueba, un operador no involucrado en la creación del playbook debe poder concluir: + +1. La IP atacante generó múltiples fallos SSH. +2. La regla `SIEM - Brute Force SSH Detectado` corresponde al comportamiento observado. +3. No hubo acceso exitoso, o si lo hubo, se trató como compromiso probable. +4. La acción de contención elegida coincide con la evidencia. +5. Las lecciones aprendidas quedaron registradas. diff --git a/docs/playbooks/sqli-response.md b/docs/playbooks/sqli-response.md new file mode 100644 index 0000000..424ae48 --- /dev/null +++ b/docs/playbooks/sqli-response.md @@ -0,0 +1,245 @@ +# Playbook de respuesta: SQLi y rutas web sensibles + +## Objetivo + +Guiar la respuesta ante intentos de inyección SQL y accesos a rutas web sensibles observados en logs de Nginx. El playbook está orientado a confirmar el intento, estimar impacto, contener el origen y definir acciones de erradicación. + +## Resumen del escenario + +| Dato | Valor | +| --- | --- | +| Alerta principal | `SIEM - Acceso a Rutas Web Sensibles` | +| Indicador SQLi | `siem.sqli_detected:true` o tag `sqli_detected` | +| Severidad esperada | `high` para rutas sensibles, variable para SQLi según respuesta HTTP | +| Regla | `rules/sensitive-paths.json` | +| Fuente | `logs-nginx*` | +| Dataset | `event.dataset:"nginx.access"` | +| Técnicas MITRE ATT&CK | `T1190 - Exploit Public-Facing Application`, `T1552.001 - Credentials In Files` | +| Script de simulación | `python3 setup/generate-test-logs.py --attack sqli` | + +## Criterios de activación + +Ejecutar este playbook si se cumple al menos una condición: + +1. Se genera la alerta `SIEM - Acceso a Rutas Web Sensibles`. +2. En `Discover` aparecen eventos con `siem.sqli_detected:true` o tag `sqli_detected`. +3. Una IP externa solicita rutas como `/admin`, `/.env`, `/wp-admin`, `/phpmyadmin`, `/config.php` o rutas equivalentes. +4. El `user_agent.original` muestra herramientas como `sqlmap`, `nikto`, `curl`, `nmap`, `gobuster` o `dirbuster`. + +## Roles mínimos + +| Rol | Responsabilidad | +| --- | --- | +| Operador SIEM | Investiga la alerta y recolecta evidencia. | +| Responsable de aplicación | Valida impacto en endpoint, código y datos. | +| Responsable de infraestructura | Aplica bloqueo de IP, rate limit o reglas WAF. | +| Responsable de seguridad | Decide rotación de secretos y escalamiento. | + +## Preparación + +1. Abrir `Kibana -> Security -> Alerts` y `Discover`. +2. Seleccionar una ventana de tiempo de los últimos 15 minutos. +3. Confirmar que el índice `logs-nginx*` contiene eventos recientes. +4. Si es una simulación, generar el escenario desde la raíz del repositorio: + +```bash +python3 setup/generate-test-logs.py --attack sqli +``` + +Esperar de 30 a 60 segundos para que los eventos lleguen a Kibana. + +## Fase 1: identificación + +1. Buscar la alerta `SIEM - Acceso a Rutas Web Sensibles` en `Kibana -> Security -> Alerts`. +2. En paralelo, abrir `Discover` y usar esta consulta para detectar SQLi: + +```text +event.dataset:"nginx.access" and (siem.sqli_detected:true or tags:"sqli_detected") +``` + +3. Si la alerta no aparece, buscar rutas sensibles: + +```text +event.dataset:"nginx.access" and (siem.sensitive_path:true or tags:"sensitive_path_access") +``` + +4. Registrar estos datos: + +| Campo | Valor a registrar | +| --- | --- | +| Hora del evento | `@timestamp` | +| IP origen | `source.ip` | +| Host o servicio | `host.name`, `service.name` si existe | +| Método HTTP | `http.request.method` | +| URL completa | `url.original` | +| Ruta | `url.path` | +| Query string | `url.query` | +| Código HTTP | `http.response.status_code` | +| User agent | `user_agent.original` | +| Tags SIEM | `tags` | + +## Fase 2: investigación + +### Paso 1: confirmar patrón de SQLi + +Revisar `url.original` y `url.query` buscando patrones como: + +| Patrón | Interpretación | +| --- | --- | +| `' OR '1'='1` | Bypass de autenticación o condición siempre verdadera. | +| `UNION SELECT` | Intento de extracción de columnas o tablas. | +| `DROP TABLE` | Intento destructivo contra base de datos. | +| `--` | Comentario SQL para alterar la consulta. | +| User agent `sqlmap` | Automatización de prueba o ataque SQLi. | + +Consulta KQL recomendada: + +```text +event.dataset:"nginx.access" and source.ip:"" and (url.original:*UNION* or url.original:*SELECT* or url.original:*DROP* or url.original:*1\=1* or user_agent.original:*sqlmap*) +``` + +### Paso 2: evaluar respuesta del servidor + +Clasificar impacto según `http.response.status_code`: + +| Código | Interpretación | Severidad operativa | +| --- | --- | --- | +| `200` | La aplicación respondió correctamente al payload o ruta sensible. Puede haber exposición. | Alta | +| `301` o `302` | Redirección. Revisar destino y si redirige a login. | Media | +| `403` | Bloqueado por aplicación, WAF o control de acceso. | Media | +| `404` | Ruta inexistente. Puede ser reconocimiento. | Baja a media | +| `500` | Error interno. Puede indicar vulnerabilidad o fallo de manejo de entrada. | Alta | + +### Paso 3: identificar rutas sensibles solicitadas + +Buscar todas las solicitudes de la misma IP en la ventana de investigación: + +```text +event.dataset:"nginx.access" and source.ip:"" +``` + +Registrar si aparecen rutas como: + +| Ruta | Riesgo | +| --- | --- | +| `/.env` | Exposición de secretos. | +| `/config.php` o `/config` | Exposición de configuración. | +| `/.git` | Exposición de código fuente. | +| `/phpmyadmin` | Administración de base de datos expuesta. | +| `/admin` o `/api/admin` | Panel administrativo expuesto. | +| `/etc/passwd` o `/proc/` | Intento de lectura de archivos del sistema. | + +### Paso 4: correlacionar actividad relacionada + +Buscar si la misma IP también generó otros indicadores: + +```text +source.ip:"" and (tags:"http_404" or tags:"security_scanner" or siem.sensitive_path:true or siem.sqli_detected:true) +``` + +Si existen muchos `404`, revisar también el playbook de enumeración web cuando esté disponible. Si existen intentos de SSH desde la misma IP, aplicar el playbook de fuerza bruta SSH. + +### Paso 5: clasificar el incidente + +| Evidencia | Clasificación | +| --- | --- | +| SQLi con `403` o `404`, sin otros eventos | Intento bloqueado o reconocimiento. | +| SQLi con `200` | Explotación posible, requiere revisión de aplicación. | +| SQLi con `500` | Vulnerabilidad probable o manejo inseguro de entrada. | +| Acceso `200` a `/.env`, `/config`, `/.git` o similar | Compromiso probable de secretos o información. | +| User agent de scanner autorizado desde IP interna documentada | Posible falso positivo, requiere evidencia de autorización. | + +## Fase 3: contención + +### Acciones para laboratorio + +1. Registrar IP origen, URL completa, código HTTP y user agent. +2. Marcar la alerta como `acknowledged` o `in progress` si Kibana lo permite. +3. No cambiar reglas de firewall productivas durante la simulación. +4. Documentar la acción que se aplicaría si fuera un entorno real. + +### Acciones para entorno real + +Aplicar según la clasificación: + +| Condición | Acción de contención | +| --- | --- | +| Scanner externo con múltiples intentos | Bloquear `source.ip` en firewall, WAF o reverse proxy. | +| Payload SQLi con respuesta `200` | Bloquear IP y poner endpoint afectado en revisión urgente. | +| Ruta sensible con respuesta `200` | Bloquear acceso público, retirar archivo/ruta y rotar secretos expuestos. | +| Error `500` provocado por payload | Activar modo mantenimiento del endpoint si hay riesgo de explotación. | +| Scanner interno autorizado | Confirmar ventana de prueba y excluir solo si está documentado. | + +Ejemplos de referencia: + +```bash +sudo ufw deny from to any port 80 +sudo ufw deny from to any port 443 +``` + +Para Nginx, una contención temporal podría ser bloquear la IP en el bloque `server` o aplicar rate limiting. Implementar cambios solo con aprobación del responsable del servicio. + +## Fase 4: erradicación + +1. Revisar el endpoint afectado y confirmar que usa consultas preparadas o parametrizadas. +2. Validar entrada en servidor para parámetros usados en búsqueda, filtros, IDs y formularios. +3. Corregir manejo de errores para evitar respuestas `500` con payloads maliciosos. +4. Remover o proteger rutas sensibles como `/.env`, `/.git`, `/config.php`, `/phpmyadmin` y `/server-status`. +5. Rotar secretos si una ruta sensible respondió `200` o si existe duda razonable de exposición. +6. Agregar pruebas de seguridad para payloads SQLi comunes. +7. Revisar reglas WAF o controles equivalentes para bloquear patrones repetidos. +8. Confirmar que nuevos intentos similares generan alerta y no devuelven `200`. + +Consulta de verificación: + +```text +event.dataset:"nginx.access" and source.ip:"" and @timestamp >= "" +``` + +## Fase 5: recuperación y monitoreo + +1. Monitorear durante 30 minutos después de aplicar contención. +2. Confirmar que no hay nuevos `200` para payloads SQLi o rutas sensibles. +3. Revisar que el tráfico legítimo al servicio continúa funcionando. +4. Cerrar la alerta solo cuando la causa raíz esté documentada o el evento esté clasificado como falso positivo. + +## Lecciones aprendidas + +Completar estas preguntas al cierre: + +| Pregunta | Respuesta | +| --- | --- | +| La detección apareció en menos de 60 segundos | | +| La regla o indicador mostró la URL completa | | +| El código HTTP permitió clasificar impacto | | +| Hubo respuesta `200` a payload SQLi o ruta sensible | | +| Se identificó el endpoint vulnerable o expuesto | | +| Se requiere WAF, validación adicional o cambio de código | | +| Se requiere rotación de secretos | | +| Qué ajuste se hará a reglas, dashboards o documentación | | + +## Checklist de cierre + +| Ítem | Estado | +| --- | --- | +| Indicador SQLi o alerta de ruta sensible identificado | | +| IP origen registrada | | +| URL y payload registrados | | +| Código HTTP evaluado | | +| User agent registrado | | +| Se revisaron otros eventos de la misma IP | | +| Se clasificó el impacto | | +| Se definió contención | | +| Se definió erradicación | | +| Se documentaron lecciones aprendidas | | +| Otro integrante pudo seguir el procedimiento | | + +## Resultado esperado en simulación + +Para considerar exitosa la prueba, un operador no involucrado en la creación del playbook debe poder concluir: + +1. La IP atacante generó payloads compatibles con SQLi y accesos a rutas sensibles. +2. Los campos `siem.sqli_detected`, `siem.sensitive_path`, `url.original` y `http.response.status_code` permiten explicar el incidente. +3. La severidad se justifica con la respuesta HTTP y el tipo de ruta solicitada. +4. La contención propuesta reduce nuevos intentos desde el origen observado. +5. Las acciones de erradicación apuntan a la causa raíz y no solo a la alerta. diff --git a/elasticsearch/config/log4j2.properties b/elasticsearch/config/log4j2.properties new file mode 100644 index 0000000..b1290cb --- /dev/null +++ b/elasticsearch/config/log4j2.properties @@ -0,0 +1,34 @@ +status = error +name = ElasticsearchPropertiesConfig + +appender.console.type = Console +appender.console.name = rolling +appender.console.layout.type = ECSJsonLayout +appender.console.layout.dataset = elasticsearch.server + +rootLogger.level = warn +rootLogger.appenderRef.console.ref = rolling + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = error + +logger.deprecation_analytics.name = org.elasticsearch.deprecation.xpack.application.analytics.action.TransportDeleteAnalyticsCollectionAction +logger.deprecation_analytics.level = off + +logger.deprecation_dotprefix.name = org.elasticsearch.deprecation.validation.DotPrefixValidator +logger.deprecation_dotprefix.level = off + +logger.entitlements_xpack_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security +logger.entitlements_xpack_security.level = error + +logger.entitlements_inference.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-inference.software.amazon.awssdk.profiles +logger.entitlements_inference.level = error + +logger.entitlements_ingest_attachment.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.ingest-attachment.ALL-UNNAMED +logger.entitlements_ingest_attachment.level = error + +logger.entitlements_repository_gcs.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.repository-gcs.ALL-UNNAMED +logger.entitlements_repository_gcs.level = error + +logger.entitlements_repository_s3.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.repository-s3.ALL-UNNAMED +logger.entitlements_repository_s3.level = error diff --git a/filebeat/filebeat-daemonset.yaml b/filebeat/filebeat-daemonset.yaml new file mode 100644 index 0000000..08a1560 --- /dev/null +++ b/filebeat/filebeat-daemonset.yaml @@ -0,0 +1,219 @@ +# ============================================================================= +# Filebeat DaemonSet — Kubernetes +# Despliega un pod de Filebeat en CADA nodo del cluster +# ============================================================================= + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + app: filebeat + project: siem +data: + filebeat.yml: |- + filebeat.inputs: + - type: filestream + id: k8s-containers + prospector.scanner.symlinks: true + paths: + - /var/log/containers/*.log + fields: + event.module: kubernetes + event.kind: event + fields_under_root: true + parsers: + - container: + stream: all + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + in_cluster: true + matchers: + - logs_path: + logs_path: /var/log/containers/ + - add_docker_metadata: + host: "unix:///var/run/docker.sock" + - add_fields: + target: ecs + fields: + version: "8.0" + - drop_fields: + fields: ["agent.ephemeral_id", "log.offset"] + ignore_missing: true + - script: + lang: javascript + id: k8s-classifier + source: > + function process(event) { + var ns = event.Get("kubernetes.namespace") || "unknown"; + var pod = event.Get("kubernetes.pod.name") || "unknown"; + var ctr = event.Get("kubernetes.container.name") || "unknown"; + event.Put("orchestrator.type", "kubernetes"); + event.Put("orchestrator.namespace", ns); + event.Put("orchestrator.resource.name", pod); + event.Put("container.name", ctr); + if (ns === "kube-system") { + event.Put("event.dataset", "kubernetes.system"); + } else { + event.Put("event.dataset", "kubernetes.container"); + } + return event; + } + + output.logstash: + hosts: ["${LOGSTASH_HOST}"] + + logging.level: warning + logging.to_files: false + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + app: filebeat + project: siem +spec: + selector: + matchLabels: + app: filebeat + template: + metadata: + labels: + app: filebeat + project: siem + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:9.3.3 + args: ["-c", "/etc/filebeat.yml", "-e"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LOGSTASH_HOST + value: "logstash.siem.svc.cluster.local:5044" + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password + securityContext: + runAsUser: 0 + capabilities: + add: + - DAC_READ_SEARCH + resources: + requests: + memory: "100Mi" + cpu: "100m" + limits: + memory: "200Mi" + cpu: "500m" + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + subPath: filebeat.yml + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlogcontainers + mountPath: /var/log/containers + readOnly: true + - name: varlogpods + mountPath: /var/log/pods + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + readOnly: true + livenessProbe: + exec: + command: + - sh + - -c + - curl -f http://localhost:5066/stats || exit 1 + initialDelaySeconds: 60 + periodSeconds: 30 + + volumes: + - name: config + configMap: + name: filebeat-config + defaultMode: 0640 + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + app: filebeat +rules: + - apiGroups: [""] + resources: + - namespaces + - pods + - nodes + verbs: ["get", "watch", "list"] + - apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + app: filebeat + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io diff --git a/filebeat/filebeat-k8s.yml b/filebeat/filebeat-k8s.yml new file mode 100644 index 0000000..0d1fd54 --- /dev/null +++ b/filebeat/filebeat-k8s.yml @@ -0,0 +1,102 @@ +# ============================================================================= +# Filebeat — Configuración para DaemonSet de Kubernetes +# Se despliega como pod en cada nodo del cluster +# ============================================================================= + +filebeat.inputs: + + # ─── Logs de todos los contenedores del nodo ────────────────────────────── + - type: filestream + id: k8s-all-containers + enabled: true + paths: + - /var/log/containers/*.log + prospector.scanner.symlinks: true + fields: + event.module: kubernetes + event.kind: event + fields_under_root: true + parsers: + - container: + stream: all + + # ─── Logs de componentes del plano de control ──────────────────────────── + - type: filestream + id: k8s-control-plane + enabled: true + paths: + - /var/log/kubernetes/*.log + - /var/log/pods/kube-system_*/*/*.log + fields: + event.dataset: kubernetes.audit + event.module: kubernetes + event.kind: event + event.category: host + fields_under_root: true + tags: ["kubernetes", "control-plane"] + +# ─── Procesadores Kubernetes ────────────────────────────────────────────────── +processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + in_cluster: true + default_indexers.enabled: true + default_matchers.enabled: true + matchers: + - logs_path: + logs_path: /var/log/containers/ + + - add_docker_metadata: + host: "unix:///var/run/docker.sock" + + - decode_json_fields: + fields: ["message"] + target: "" + overwrite_keys: true + add_error_key: true + max_depth: 2 + + - add_host_metadata: + netinfo.enabled: true + + - add_fields: + target: ecs + fields: + version: "8.0" + + # Mapear namespace a event.category + - script: + lang: javascript + id: k8s-namespace-classifier + source: > + function process(event) { + var ns = event.Get("kubernetes.namespace") || ""; + if (ns === "kube-system") { + event.Put("event.category", "host"); + event.Put("event.dataset", "kubernetes.system"); + } else if (ns.indexOf("monitoring") !== -1) { + event.Put("event.category", "host"); + event.Put("event.dataset", "kubernetes.monitoring"); + } else { + event.Put("event.category", "host"); + event.Put("event.dataset", "kubernetes.container"); + } + event.Put("orchestrator.type", "kubernetes"); + return event; + } + + - drop_fields: + fields: ["agent.ephemeral_id", "log.offset"] + ignore_missing: true + +output.logstash: + hosts: ["${LOGSTASH_HOST:logstash:5044}"] + worker: 2 + +setup.kibana: + host: "${KIBANA_HOST:https://kibana:5601}" + ssl.enabled: true + ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] + +logging.level: warning +logging.to_files: false # En k8s los logs van a stdout diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml new file mode 100644 index 0000000..c7f8145 --- /dev/null +++ b/filebeat/filebeat.yml @@ -0,0 +1,291 @@ +# ============================================================================= +# Filebeat — Configuración principal SIEM +# Fuentes: syslog, nginx/apache, auth.log, Kubernetes (Docker containers) +# Mapeo: Elastic Common Schema (ECS) 8.x +# ============================================================================= + +filebeat.inputs: + + # ─── 1. SYSLOG del sistema ───────────────────────────────────────────────── + - type: filestream + id: syslog-system + enabled: true + paths: + - /var/log/syslog + - /var/log/messages + - /var/log/kern.log + encoding: utf-8 + # ECS fields + fields: + event.dataset: system.syslog + event.module: system + event.kind: event + event.category: host + fields_under_root: true + # Multiline: una entrada de log puede abarcar varias líneas + parsers: + - multiline: + type: pattern + pattern: '^\w{3}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}' + negate: true + match: after + # Procesadores específicos de esta fuente + processors: + - add_fields: + target: '' + fields: + log.type: syslog + - convert: + fields: + - {from: "process.pid", type: integer} + ignore_missing: true + fail_on_error: false + + # ─── 2. AUTH LOG (SSH, sudo, PAM, cron) ──────────────────────────────────── + - type: filestream + id: auth-log + enabled: true + paths: + - /var/log/auth.log + - /var/log/secure # RHEL/CentOS + - /var/log/audit/audit.log # auditd (opcional) + encoding: utf-8 + fields: + event.dataset: system.auth + event.module: system + event.kind: event + event.category: authentication + fields_under_root: true + # Tags para identificación rápida en Logstash + tags: ["auth", "security"] + processors: + - add_fields: + target: '' + fields: + log.type: auth + # Extraer acción básica del mensaje antes de enviarlo a Logstash + - script: + lang: javascript + id: auth-classifier + source: > + function process(event) { + var msg = event.Get("message") || ""; + if (msg.indexOf("Failed password") !== -1 || + msg.indexOf("authentication failure") !== -1) { + event.Put("event.outcome", "failure"); + event.Put("event.action", "authentication_failure"); + event.AppendTo("tags", "auth_failure"); + } else if (msg.indexOf("Accepted") !== -1 || + msg.indexOf("session opened") !== -1) { + event.Put("event.outcome", "success"); + event.Put("event.action", "authentication_success"); + event.AppendTo("tags", "auth_success"); + } else if (msg.indexOf("Invalid user") !== -1 || + msg.indexOf("illegal user") !== -1) { + event.Put("event.outcome", "failure"); + event.Put("event.action", "invalid_user"); + event.AppendTo("tags", "auth_failure"); + } else if (msg.indexOf("COMMAND") !== -1) { + event.Put("event.action", "sudo_command"); + event.AppendTo("tags", "privilege_escalation"); + } + return event; + } + + # ─── 3. NGINX — Access log ───────────────────────────────────────────────── + - type: filestream + id: nginx-access + enabled: true + paths: + - /var/log/nginx/access.log + - /var/log/nginx/*access*.log + - /var/log/apache2/access.log + encoding: utf-8 + fields: + event.dataset: nginx.access + event.module: nginx + event.kind: event + event.category: web + event.type: access + fields_under_root: true + tags: ["nginx", "web", "access"] + processors: + - add_fields: + target: '' + fields: + log.type: nginx_access + # Clasificar por código de respuesta + - script: + lang: javascript + id: http-classifier + source: > + function process(event) { + var msg = event.Get("message") || ""; + var m = msg.match(/" (\d{3}) /); + if (m) { + var code = parseInt(m[1]); + event.Put("http.response.status_code", code); + if (code >= 500) { event.Put("event.type", "error"); } + else if (code >= 400) { event.Put("event.type", "denied"); } + else { event.Put("event.type", "allowed"); } + } + return event; + } + + # ─── 4. NGINX — Error log ────────────────────────────────────────────────── + - type: filestream + id: nginx-error + enabled: true + paths: + - /var/log/nginx/error.log + - /var/log/nginx/*error*.log + - /var/log/apache2/error.log + encoding: utf-8 + fields: + event.dataset: nginx.error + event.module: nginx + event.kind: event + event.category: web + event.type: error + fields_under_root: true + tags: ["nginx", "web", "error"] + parsers: + - multiline: + type: pattern + pattern: '^\d{4}/\d{2}/\d{2}' + negate: true + match: after + + # ─── 5. KUBERNETES — Containers via Docker socket ────────────────────────── + - type: filestream + id: k8s-containers + enabled: true + paths: + - /var/lib/docker/containers/*/*.log + fields: + event.dataset: kubernetes.container + event.module: kubernetes + event.kind: event + event.category: host + fields_under_root: true + tags: ["kubernetes", "container"] + parsers: + - container: + stream: all # stdout + stderr + processors: + # Enriquecer con metadata de Kubernetes (requiere acceso al API server) + - add_kubernetes_metadata: + host: ${NODE_NAME:localhost} + in_cluster: false + kube_config: ${KUBECONFIG:/etc/filebeat/kubeconfig} + default_indexers.enabled: true + default_matchers.enabled: true + matchers: + - logs_path: + logs_path: /var/lib/docker/containers/ + # Enriquecer con metadata de Docker + - add_docker_metadata: + host: "unix:///var/run/docker.sock" + match_fields: + - "container.id" + # Mapear labels de k8s a campos ECS + - script: + lang: javascript + id: k8s-ecs-mapper + source: > + function process(event) { + var pod = event.Get("kubernetes.pod.name"); + var ns = event.Get("kubernetes.namespace"); + var node = event.Get("kubernetes.node.name"); + var ctr = event.Get("kubernetes.container.name"); + if (pod) event.Put("orchestrator.resource.name", pod); + if (ns) event.Put("orchestrator.namespace", ns); + if (node) event.Put("orchestrator.cluster.name", node); + if (ctr) event.Put("container.name", ctr); + event.Put("orchestrator.type", "kubernetes"); + return event; + } + + # ─── 6. KUBERNETES — Simulated container logs ────────────────────────────── + - type: filestream + id: k8s-simulated-containers + enabled: true + paths: + - /var/log/containers/*.log + encoding: utf-8 + fields: + event.dataset: kubernetes.container + event.module: kubernetes + event.kind: event + event.category: host + fields_under_root: true + tags: ["kubernetes", "container", "simulated"] + parsers: + - ndjson: + target: "" + add_error_key: true + message_key: message + processors: + - add_fields: + target: '' + fields: + log.type: k8s_container + - script: + lang: javascript + id: k8s-simulated-ecs-mapper + source: > + function process(event) { + var pod = event.Get("kubernetes.pod.name"); + var ns = event.Get("kubernetes.namespace"); + var ctr = event.Get("kubernetes.container.name"); + if (pod) event.Put("orchestrator.resource.name", pod); + if (ns) event.Put("orchestrator.namespace", ns); + if (ctr) event.Put("container.name", ctr); + event.Put("orchestrator.type", "kubernetes"); + return event; + } + +# ─── Módulos de Filebeat ───────────────────────────────────────────────────── +filebeat.modules: [] + +# ─── Output: Logstash (enriquecimiento y routing) ──────────────────────────── +output.logstash: + hosts: + - "${LOGSTASH_HOST:logstash:5044}" + worker: 2 + bulk_max_size: 2048 + compression_level: 3 + +# ─── Setup (Kibana dashboards + ILM) ───────────────────────────────────────── +setup.kibana: + host: "${KIBANA_HOST:https://kibana:5601}" + username: "${ELASTIC_USER:elastic}" + password: "${ELASTIC_PASSWORD}" + ssl.enabled: true + ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] + +setup.ilm.enabled: false +setup.dashboards.enabled: false + +# ─── Template de índice ─────────────────────────────────────────────────────── +setup.template.enabled: false +setup.template.name: "filebeat" +setup.template.pattern: "filebeat-*" +setup.template.overwrite: true + +# ─── Logging de Filebeat ────────────────────────────────────────────────────── +logging.level: warning +logging.to_files: true +logging.files: + path: /var/log/filebeat + name: filebeat + keepfiles: 7 + permissions: 0600 + +# ─── Métricas internas (monitoring) ────────────────────────────────────────── +monitoring.enabled: false + +# ─── HTTP endpoint (healthcheck) ───────────────────────────────────────────── +http.enabled: true +http.host: "0.0.0.0" +http.port: 5066 diff --git a/kibana/kibana.yml b/kibana/kibana.yml new file mode 100644 index 0000000..95effcb --- /dev/null +++ b/kibana/kibana.yml @@ -0,0 +1,36 @@ +server.name: kibana +server.host: "0.0.0.0" +server.port: 5601 + +# TLS de entrada para asegurar acceso HTTPS en Kibana +server.ssl.enabled: true +server.ssl.certificate: /usr/share/kibana/config/certs/kibana/kibana.crt +server.ssl.key: /usr/share/kibana/config/certs/kibana/kibana.key + +# Conexión al cluster +elasticsearch.hosts: ["https://es01:9200", "https://es02:9200", "https://es03:9200"] +elasticsearch.username: "kibana_system" +elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca/ca.crt"] +elasticsearch.ssl.verificationMode: certificate + +# No usamos Stack Monitoring ni la colección de métricas de Kibana en este stack +monitoring.kibana.collection.enabled: false +monitoring.ui.enabled: false + +# Security Solution (SIEM) habilitado para usarlo en el stack local +xpack.securitySolution.enabled: true + +# Reporting no se usa aquí y su arranque puede fallar bajo carga del cluster +xpack.reporting.enabled: false + +# Telemetry/usage collection desactivado en el stack local +telemetry.optIn: false +telemetry.allowChangingOptInStatus: false + +# Evita ruido de arranque; conserva WARN+ y ERROR +logging.root.level: warn + +# Silencia el task de Cases que falla cuando no hay sesiones/casos activos +logging.loggers: + - name: plugins.cases.incremental_id_task + level: error diff --git a/kibana/node.options b/kibana/node.options new file mode 100644 index 0000000..d3824b8 --- /dev/null +++ b/kibana/node.options @@ -0,0 +1,10 @@ +## Node command line options +## See `node --help` and `node --v8-options` for available options +## Please note you should specify one option per line + +## max size of old space in megabytes +#--max-old-space-size=4096 + +## restore < Node 16 default DNS lookup behavior +--dns-result-order=ipv4first + diff --git a/logstash/config/log4j2.properties b/logstash/config/log4j2.properties new file mode 100644 index 0000000..969ab8f --- /dev/null +++ b/logstash/config/log4j2.properties @@ -0,0 +1,11 @@ +status = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-40c] %m%n + +rootLogger.level = warn +rootLogger.appenderRef.console.ref = plain_console diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml new file mode 100644 index 0000000..e69de29 diff --git a/logstash/patterns/siem-patterns b/logstash/patterns/siem-patterns new file mode 100644 index 0000000..c143eb9 --- /dev/null +++ b/logstash/patterns/siem-patterns @@ -0,0 +1,28 @@ +# ============================================================================= +# Patrones Grok custom — SIEM Platform +# Guardar en: /usr/share/logstash/patterns/siem-patterns +# ============================================================================= + +SIEM_HOST (?:%{IP}|%{HOSTNAME}) +SIEM_SYSLOG_TS (?:%{MONTH} +%{MONTHDAY} %{TIME}(?:\s+%{YEAR})?) +SIEM_PROC_PID (?:\[%{POSINT}\])? +SIEM_SYSLOG_HEADER %{SIEM_SYSLOG_TS:syslog_timestamp} %{SIEM_HOST:host_name} %{DATA:process_name}%{SIEM_PROC_PID}:? + +NGINX_BYTES (?:%{NUMBER}|-) +NGINX_STATUS_CODE [1-9][0-9]{2} +NGINX_ACCESS %{IPORHOST:client_ip} - %{DATA:auth_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http_method} %{URIPATH:uri_path}(?:\?%{URIQUERY:uri_query})?(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_request})" %{NGINX_STATUS_CODE:status_code:int} %{NGINX_BYTES:bytes_sent:int} "(?:%{URI:referrer}|-)" "(?:%{DATA:user_agent}|-)" + +NGINX_ERROR_SEVERITY emerg|alert|crit|error|warn|notice|info|debug +NGINX_ERROR %{DATA:error_time} \[%{NGINX_ERROR_SEVERITY:severity}\] %{NUMBER:pid:int}#%{NUMBER:tid:int}: (?:\*%{NUMBER:connection_id:int} )?%{DATA:error_msg}(?:, client: %{IPORHOST:client_ip})?(?:, server: %{HOSTNAME:server_name})? + +SSH_AUTH_METHOD publickey|password|keyboard-interactive|gssapi-with-mic|hostbased|none +SSH_ACCEPTED %{SIEM_SYSLOG_HEADER} Accepted %{SSH_AUTH_METHOD:auth_method} for %{USERNAME:username} from %{IPORHOST:src_ip} port %{NUMBER:src_port:int} +SSH_FAILED_VALID %{SIEM_SYSLOG_HEADER} Failed password for %{USERNAME:username} from %{IPORHOST:src_ip} port %{NUMBER:src_port:int} +SSH_FAILED_INVALID %{SIEM_SYSLOG_HEADER} Failed password for invalid user %{USERNAME:username} from %{IPORHOST:src_ip} port %{NUMBER:src_port:int} +SSH_MAX_ATTEMPTS %{SIEM_SYSLOG_HEADER} error: maximum authentication attempts exceeded for %{DATA:username} from %{IPORHOST:src_ip} port %{NUMBER:src_port:int} + +UFW_PROTO TCP|UDP|ICMP|GRE|ESP|AH +UFW_BLOCK %{SIEM_SYSLOG_HEADER} kernel: \[%{DATA:kernel_time}\] \[%{DATA:ufw_rule}\] IN=%{DATA:in_iface} OUT=%{DATA:out_iface}(?: MAC=%{DATA:mac_addr})? SRC=%{IPORHOST:src_ip} DST=%{IPORHOST:dst_ip}(?: LEN=%{NUMBER:pkt_len:int})?(?: TTL=%{NUMBER:ttl:int})? PROTO=%{UFW_PROTO:proto}(?: SPT=%{NUMBER:src_port:int})?(?: DPT=%{NUMBER:dst_port:int})? + +APP_LOG_JSON \{"time":"%{TIMESTAMP_ISO8601:log_time}","level":"%{LOGLEVEL:log_level}","msg":"%{DATA:log_msg}"%{DATA:log_extra}\} +APP_LOG_ISO %{TIMESTAMP_ISO8601:log_time} \[?%{LOGLEVEL:log_level}\]? %{DATA:logger_name}(?::%{NUMBER:line:int})? - %{GREEDYDATA:log_msg} diff --git a/logstash/pipeline/01-input.conf b/logstash/pipeline/01-input.conf new file mode 100644 index 0000000..6d7cea9 --- /dev/null +++ b/logstash/pipeline/01-input.conf @@ -0,0 +1,39 @@ +# ============================================================================= +# PIPELINE 01 — INPUT +# Recibe eventos de Filebeat (puerto 5044) y Syslog directo (puerto 5514) +# ============================================================================= + +input { + + # ── Filebeat ─────────────────────────────────────────────────────────────── + beats { + port => 5044 + host => "0.0.0.0" + # client_inactivity_timeout: tiempo sin datos antes de cerrar conexión + client_inactivity_timeout => 60 + # Añadir tag para saber que entró por este input + add_field => { "[@metadata][input]" => "beats" } + } + + # ── Syslog UDP (dispositivos de red, servidores legacy) ─────────────────── + udp { + port => 5514 + type => "syslog-udp" + codec => plain { charset => "UTF-8" } + add_field => { + "[@metadata][input]" => "syslog-udp" + "event.dataset" => "system.syslog" + } + } + + # ── Syslog TCP ──────────────────────────────────────────────────────────── + tcp { + port => 5514 + type => "syslog-tcp" + codec => line { charset => "UTF-8" } + add_field => { + "[@metadata][input]" => "syslog-tcp" + "event.dataset" => "system.syslog" + } + } +} diff --git a/logstash/pipeline/02-preprocess.conf b/logstash/pipeline/02-preprocess.conf new file mode 100644 index 0000000..05ad498 --- /dev/null +++ b/logstash/pipeline/02-preprocess.conf @@ -0,0 +1,61 @@ +# ============================================================================= +# PIPELINE 02 — PRE-PROCESAMIENTO +# Normalización de timestamp, routing por dataset, campos base ECS +# Se ejecuta ANTES de los filtros específicos por fuente +# ============================================================================= + +filter { + + # ── 1. Añadir campos ECS base a todos los eventos ───────────────────────── + mutate { + add_field => { + "ecs.version" => "8.0" + "[@metadata][index]" => "unknown" # se sobreescribe en cada filtro + } + } + + # ── 1.1 Limpieza de bytes nulos para evitar fallos de grok ─────────────── + if [message] { + mutate { gsub => ["message", "\u0000", ""] } + } + if [event][original] { + mutate { gsub => ["[event][original]", "\u0000", ""] } + } + + # ── 2. Inferir dataset desde campos de Filebeat si no viene explícito ────── + if ![event][dataset] { + if [type] == "syslog-udp" or [type] == "syslog-tcp" { + mutate { add_field => { "[event][dataset]" => "system.syslog" } } + } else if [log][file][path] =~ /auth\.log|secure$/ { + mutate { add_field => { "[event][dataset]" => "system.auth" } } + } else if [log][file][path] =~ /nginx/ and [log][file][path] =~ /access/ { + mutate { add_field => { "[event][dataset]" => "nginx.access" } } + } else if [log][file][path] =~ /nginx/ and [log][file][path] =~ /error/ { + mutate { add_field => { "[event][dataset]" => "nginx.error" } } + } else if [log][file][path] =~ /apache/ { + mutate { add_field => { "[event][dataset]" => "nginx.access" } } + } else if [kubernetes] or [container] { + mutate { add_field => { "[event][dataset]" => "kubernetes.container" } } + } else if [log][file][path] =~ /syslog|messages|kern/ { + mutate { add_field => { "[event][dataset]" => "system.syslog" } } + } + } + + # ── 3. Establecer el índice de destino por dataset ───────────────────────── + if [event][dataset] == "nginx.access" or [event][dataset] == "nginx.error" { + mutate { replace => { "[@metadata][index]" => "logs-nginx" } } + } else if [event][dataset] == "system.auth" { + mutate { replace => { "[@metadata][index]" => "logs-auth" } } + } else if [event][dataset] == "system.syslog" { + mutate { replace => { "[@metadata][index]" => "logs-syslog" } } + } else if [event][dataset] =~ /^kubernetes/ { + mutate { replace => { "[@metadata][index]" => "logs-k8s" } } + } + + # ── 4. Copiar log.file.path a un campo visible ──────────────────────────── + if [log][file][path] { + mutate { + copy => { "[log][file][path]" => "[@metadata][source_file]" } + } + } +} diff --git a/logstash/pipeline/03-filter-nginx.conf b/logstash/pipeline/03-filter-nginx.conf new file mode 100644 index 0000000..343bd9b --- /dev/null +++ b/logstash/pipeline/03-filter-nginx.conf @@ -0,0 +1,214 @@ +# ============================================================================= +# PIPELINE 03 — FILTRO NGINX +# Parsea access.log y error.log de Nginx/Apache con grok +# Enriquece con GeoIP, User-Agent, clasificación de ataques +# Tasa de éxito objetivo: > 95% +# ============================================================================= + +filter { + + # ════════════════════════════════════════════════════════════════════════════ + # NGINX ACCESS LOG + # ════════════════════════════════════════════════════════════════════════════ + if [event][dataset] == "nginx.access" { + + # ── Grok: formato combined (default nginx) ────────────────────────────── + # Ejemplo de línea: + # 185.220.101.34 - admin [25/Apr/2024:14:32:01 +0000] "GET /api/v1/users HTTP/1.1" 200 1842 "-" "curl/7.68.0" + grok { + match => { + "message" => [ + # Formato combined completo (con referrer y user agent) + '%{IPORHOST:[source][ip]} - %{DATA:[user][name]} \[%{HTTPDATE:[nginx][access][time]}\] "(?:%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]}|-)" %{NUMBER:[http][response][status_code]:int} (?:%{NUMBER:[http][response][body][bytes]:int}|-) "(?:%{DATA:[http][request][referrer]}|-)" "(?:%{DATA:[user_agent][original]}|-)"', + + # Formato combined sin referrer/UA + '%{IPORHOST:[source][ip]} - %{DATA:[user][name]} \[%{HTTPDATE:[nginx][access][time]}\] "(?:%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]}|-)" %{NUMBER:[http][response][status_code]:int} (?:%{NUMBER:[http][response][body][bytes]:int}|-)', + + # Formato con X-Forwarded-For (proxy/CDN) + '%{IPORHOST:[source][ip]}, %{IPORHOST:[network][forwarded_ip]} - %{DATA:[user][name]} \[%{HTTPDATE:[nginx][access][time]}\] "(?:%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]}|-)" %{NUMBER:[http][response][status_code]:int} (?:%{NUMBER:[http][response][body][bytes]:int}|-) "(?:%{DATA:[http][request][referrer]}|-)" "(?:%{DATA:[user_agent][original]}|-)"' + ] + } + tag_on_failure => ["_grokparsefailure_nginx_access"] + } + + # Sólo continuar si el parseo fue exitoso + if "_grokparsefailure_nginx_access" not in [tags] { + + # ── Parsear timestamp nginx ─────────────────────────────────────────── + date { + match => [ "[nginx][access][time]", "dd/MMM/yyyy:HH:mm:ss Z" ] + target => "@timestamp" + timezone => "America/Bogota" + tag_on_failure => ["_dateparsefailure_nginx"] + } + + # ── Extraer URL path y query string ────────────────────────────────── + if [url][original] { + grok { + match => { + "[url][original]" => "^%{URIPATH:[url][path]}(?:\?%{URIQUERY:[url][query]})?$" + } + tag_on_failure => [] # fallo silencioso — no siempre hay query + } + } + + # ── Calcular duración de respuesta (si nginx lo incluye — $request_time) + # Descomenta si tu nginx.conf incluye $request_time en el log format: + # mutate { convert => { "[nginx][access][response_time]" => "float" } } + + # ── Clasificar evento HTTP por código de respuesta ──────────────────── + mutate { + add_field => { + "[event][kind]" => "event" + "[event][category]" => "web" + "[event][module]" => "nginx" + } + } + + # Tipo según código de respuesta + if [http][response][status_code] { + if [http][response][status_code] >= 500 { + mutate { + add_field => { "[event][type]" => "error" } + add_tag => ["http_5xx"] + } + } else if [http][response][status_code] >= 400 { + mutate { + add_field => { "[event][type]" => "denied" } + } + # 404 específico — importante para regla de detección R8.4 + if [http][response][status_code] == 404 { + mutate { add_tag => ["http_404"] } + } + # 401/403 — acceso denegado + if [http][response][status_code] == 401 or [http][response][status_code] == 403 { + mutate { + add_tag => ["access_denied"] + add_field => { "[event][outcome]" => "failure" } + } + } + } else if [http][response][status_code] >= 300 { + mutate { add_field => { "[event][type]" => "redirect" } } + } else { + mutate { + add_field => { "[event][type]" => "allowed" } + add_field => { "[event][outcome]" => "success" } + } + } + } + + # ── Detección de rutas sensibles (R8.4 — regla acceso a rutas sensibles) + if [url][path] { + if [url][path] =~ /(?i)(\/admin|\/wp-admin|\/phpmyadmin|\/\.env|\/config|\/backup|\/\.git|\/etc\/passwd|\/proc\/|\/server-status|\/actuator|\/api\/admin)/ { + mutate { + add_tag => ["sensitive_path_access"] + add_field => { "[siem][sensitive_path]" => true } + } + } + } + + # ── Detección de inyección SQL en URL (R8.5 — escenario SQLi) ───────── + if [url][query] or [url][original] { + # Combinar path+query para análisis + mutate { + add_field => { "[siem][url_full]" => "%{[url][original]}" } + } + + if [siem][url_full] =~ /(?i)(\bunion\b.*\bselect\b|\bselect\b.*\bfrom\b|\binsert\b.*\binto\b|\bdelete\b.*\bfrom\b|\bdrop\b.*\btable\b|\bexec\b\s*\(|\/\*.*\*\/|--\s|'\s*or\s*'|'\s*and\s*'|1=1|0x[0-9a-f]{4}|char\([0-9]+\))/ { + mutate { + add_tag => ["sqli_detected"] + add_field => { + "[siem][sqli_detected]" => true + "[event][category]" => "intrusion_detection" + "[threat][technique][name]" => "SQL Injection" + "[threat][tactic][name]" => "Initial Access" + } + } + } + + # ── Detección XSS ───────────────────────────────────────────────── + if [siem][url_full] =~ /(?i)(]|javascript:|on(?:load|error|click|mouse)\s*=|alert\s*\(|document\.cookie|eval\s*\()/ { + mutate { + add_tag => ["xss_detected"] + add_field => { + "[siem][xss_detected]" => true + "[threat][technique][name]" => "Cross-Site Scripting" + } + } + } + + # ── Detección Path Traversal ────────────────────────────────────── + if [siem][url_full] =~ /(?:\.\.\/|\.\.\\|%2e%2e%2f|%252e%252e%252f)/ { + mutate { + add_tag => ["path_traversal"] + add_field => { "[siem][path_traversal]" => true } + } + } + + mutate { remove_field => ["[siem][url_full]"] } + } + + # ── Detección de scanners y herramientas de ataque por User-Agent ────── + if [user_agent][original] { + if [user_agent][original] =~ /(?i)(nikto|sqlmap|nmap|masscan|zgrab|dirbuster|gobuster|wfuzz|hydra|medusa|metasploit|nessus|openvas|nuclei|burpsuite|zap)/ { + mutate { + add_tag => ["security_scanner"] + add_field => { "[siem][scanner_detected]" => true } + } + } + } + + # ── Mutate: limpiar campo temporal de nginx ─────────────────────────── + mutate { + remove_field => [ "[nginx][access][time]" ] + # Convertir tipos correctamente + convert => { + "[http][response][status_code]" => "integer" + "[http][response][body][bytes]" => "integer" + } + } + + # Normalizar IP vacía a guión + if [user][name] == "-" { + mutate { replace => { "[user][name]" => "" } } + } + } + } + + + # ════════════════════════════════════════════════════════════════════════════ + # NGINX ERROR LOG + # ════════════════════════════════════════════════════════════════════════════ + if [event][dataset] == "nginx.error" { + + # Ejemplo de línea nginx error: + # 2024/04/25 14:32:01 [error] 1234#1234: *56 connect() failed (111: Connection refused) while connecting to upstream, client: 10.0.0.5, server: api.example.com, request: "GET /api/v1/health HTTP/1.1", upstream: "http://backend:8080/api/v1/health", host: "api.example.com" + grok { + match => { + "message" => [ + '%{DATA:[nginx][error][time]} \[%{DATA:[log][level]}\] %{NUMBER:[process][pid]:int}#%{NUMBER:[process][tid]:int}: (?:\*%{NUMBER:[nginx][error][connection_id]:int} )?%{DATA:[error][message]}(?:, client: %{IPORHOST:[source][ip]})?(?:, server: %{DATA:[server][domain]})?(?:, request: "%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]}")?(?:, upstream: "%{DATA:[nginx][error][upstream]}")?(?:, host: "%{DATA:[server][domain2]}")?$' + ] + } + tag_on_failure => ["_grokparsefailure_nginx_error"] + } + + if "_grokparsefailure_nginx_error" not in [tags] { + date { + match => [ "[nginx][error][time]", "yyyy/MM/dd HH:mm:ss" ] + target => "@timestamp" + timezone => "America/Bogota" + tag_on_failure => ["_dateparsefailure_nginx_error"] + } + + mutate { + add_field => { + "[event][kind]" => "event" + "[event][category]" => "web" + "[event][type]" => "error" + "[event][module]" => "nginx" + } + remove_field => ["[nginx][error][time]", "[server][domain2]"] + } + } + } +} diff --git a/logstash/pipeline/04-filter-auth.conf b/logstash/pipeline/04-filter-auth.conf new file mode 100644 index 0000000..4a07c85 --- /dev/null +++ b/logstash/pipeline/04-filter-auth.conf @@ -0,0 +1,281 @@ +# ============================================================================= +# PIPELINE 04 — FILTRO AUTH.LOG +# Parsea /var/log/auth.log (Debian/Ubuntu) y /var/log/secure (RHEL/CentOS) +# Detecta: SSH brute force, sudo escalation, PAM failures, cron auth +# Enriquece con GeoIP y campos SIEM para correlación de eventos +# ============================================================================= + +filter { + + if [event][dataset] == "system.auth" { + + # ════════════════════════════════════════════════════════════════════════ + # GROK PATTERNS — orden de más específico a más general + # ════════════════════════════════════════════════════════════════════════ + + # ── Patrón 1: SSH — intento fallido con usuario válido ────────────────── + # Failed password for root from 185.220.101.34 port 54321 ssh2 + if !("_grokparsefailure" in [tags]) and [message] =~ /Failed password for / { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Failed password for %{DATA:[user][name]} from %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int} %{GREEDYDATA:[auth][method]}" + } + tag_on_failure => ["_grokparsefailure_auth_failed_pw"] + } + mutate { + add_field => { + "[event][action]" => "authentication_failure" + "[event][outcome]" => "failure" + "[event][category]" => "authentication" + "[event][type]" => "start" + "[siem][brute_force_attempt]" => true + } + add_tag => ["auth_failure", "ssh_failure"] + } + } + + # ── Patrón 2: SSH — intento fallido con usuario INVÁLIDO ───────────────── + # Failed password for invalid user ftp from 89.248.167.131 port 49823 ssh2 + else if !("_grokparsefailure" in [tags]) and [message] =~ /invalid user/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Failed password for invalid user %{DATA:[user][name]} from %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int} %{GREEDYDATA:[auth][method]}" + } + tag_on_failure => ["_grokparsefailure_auth_invalid_user"] + } + mutate { + add_field => { + "[event][action]" => "authentication_failure" + "[event][outcome]" => "failure" + "[event][category]" => "authentication" + "[siem][brute_force_attempt]" => true + "[siem][invalid_user]" => true + } + add_tag => ["auth_failure", "ssh_failure", "invalid_user"] + } + } + + # ── Patrón 3: SSH — login exitoso (Accepted) ────────────────────────── + # Accepted publickey for ubuntu from 192.168.1.20 port 52341 ssh2 + # Accepted password for deploy from 10.0.0.5 port 44123 ssh2 + else if !("_grokparsefailure" in [tags]) and [message] =~ /Accepted / { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Accepted %{DATA:[auth][method]} for %{DATA:[user][name]} from %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int}" + } + tag_on_failure => ["_grokparsefailure_auth_accepted"] + } + mutate { + add_field => { + "[event][action]" => "authentication_success" + "[event][outcome]" => "success" + "[event][category]" => "authentication" + "[event][type]" => "start" + } + add_tag => ["auth_success", "ssh_login"] + } + } + + # ── Patrón 4: SSH — desconexión / cierre de sesión ──────────────────── + # Disconnected from user ubuntu 192.168.1.20 port 52341 + # Connection closed by 185.220.101.34 port 54321 + else if !("_grokparsefailure" in [tags]) and [message] =~ /Disconnect|Connection closed/ { + grok { + match => { + "message" => [ + "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Disconnected from (?:user %{DATA:[user][name]} )?%{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int}", + "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Connection closed by %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int}" + ] + } + tag_on_failure => ["_grokparsefailure_auth_disconnect"] + } + mutate { + add_field => { + "[event][action]" => "session_end" + "[event][category]" => "session" + "[event][type]" => "end" + } + add_tag => ["ssh_disconnect"] + } + } + + # ── Patrón 5: SSH — máximo de intentos de autenticación excedido ───── + # error: maximum authentication attempts exceeded for root from 185.220.101.34 port 54321 + else if !("_grokparsefailure" in [tags]) and [message] =~ /maximum authentication attempts/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: error: maximum authentication attempts exceeded for %{DATA:[user][name]} from %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int}" + } + tag_on_failure => ["_grokparsefailure_auth_maxattempts"] + } + mutate { + add_field => { + "[event][action]" => "authentication_failure" + "[event][outcome]" => "failure" + "[event][category]" => "authentication" + "[siem][max_attempts_exceeded]" => true + "[siem][brute_force_attempt]" => true + } + add_tag => ["auth_failure", "max_attempts_exceeded"] + } + } + + # ── Patrón 6: SUDO — ejecución de comando con privilegios ──────────── + # ubuntu : TTY=pts/0 ; PWD=/home/ubuntu ; USER=root ; COMMAND=/bin/bash + else if !("_grokparsefailure" in [tags]) and [message] =~ /sudo.*COMMAND=/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} sudo\s*(?:\[%{POSINT:[process][pid]:int}\])?\s*:\s*%{DATA:[user][name]}\s*:\s*TTY=%{DATA:[process][tty]}\s*;\s*PWD=%{DATA:[process][working_directory]}\s*;\s*USER=%{DATA:[user][effective][name]}\s*;\s*COMMAND=%{GREEDYDATA:[process][command_line]}" + } + tag_on_failure => ["_grokparsefailure_sudo"] + } + mutate { + add_field => { + "[event][action]" => "sudo_command" + "[event][category]" => "process" + "[event][type]" => "start" + "[event][outcome]" => "success" + } + add_tag => ["sudo_command", "privilege_escalation"] + } + + # Comandos de alto riesgo vía sudo + if [process][command_line] =~ /(?:\/bin\/bash|\/bin\/sh|\/usr\/bin\/python|\/usr\/bin\/perl|\/usr\/bin\/nc|netcat|chmod 777|chown root|passwd|visudo|\/etc\/sudoers)/ { + mutate { + add_tag => ["high_risk_sudo"] + add_field => { "[siem][high_risk_command]" => true } + } + } + } + + # ── Patrón 7: SUDO — fallo de autenticación (contraseña incorrecta) ── + # pam_unix(sudo:auth): authentication failure; logname= uid=1000 euid=0 tty=/dev/pts/0 ruser=ubuntu rhost= user=ubuntu + else if !("_grokparsefailure" in [tags]) and [message] =~ /sudo.*authentication failure/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: pam_unix\(sudo:auth\): authentication failure;.*uid=%{NUMBER:[user][id]}.*user=%{DATA:[user][name]}" + } + tag_on_failure => ["_grokparsefailure_sudo_fail"] + } + mutate { + add_field => { + "[event][action]" => "sudo_failure" + "[event][category]" => "authentication" + "[event][outcome]" => "failure" + } + add_tag => ["sudo_failure"] + } + } + + # ── Patrón 8: PAM — apertura/cierre de sesión ───────────────────────── + # pam_unix(sshd:session): session opened for user ubuntu by (uid=0) + # pam_unix(sshd:session): session closed for user ubuntu + else if !("_grokparsefailure" in [tags]) and [message] =~ /pam_unix.*session (opened|closed)/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: pam_unix\(%{DATA:[pam][service]}:session\): session %{DATA:[pam][session_action]} for user %{DATA:[user][name]}" + } + tag_on_failure => ["_grokparsefailure_pam_session"] + } + mutate { + add_field => { + "[event][action]" => "session_%{[pam][session_action]}" + "[event][category]" => "session" + } + } + if [pam][session_action] == "opened" { + mutate { add_field => { "[event][type]" => "start" } } + } else { + mutate { add_field => { "[event][type]" => "end" } } + } + } + + # ── Patrón 9: SSH — escucha nueva conexión (informativo) ───────────── + # sshd: Server listening on 0.0.0.0 port 22. + else if !("_grokparsefailure" in [tags]) and [message] =~ /Server listening/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Server listening on %{IPORHOST:[server][ip]} port %{NUMBER:[server][port]:int}" + } + tag_on_failure => [] + } + mutate { + add_field => { + "[event][action]" => "server_listening" + "[event][category]" => "network" + "[event][type]" => "start" + } + } + } + + # ── Patrón 10: Genérico — captura todo lo demás de auth.log ────────── + else { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[auth][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYDATA:[auth][message_text]}" + } + tag_on_failure => ["_grokparsefailure_auth_generic"] + } + mutate { + add_field => { + "[event][category]" => "authentication" + "[event][kind]" => "event" + } + } + } + + # ════════════════════════════════════════════════════════════════════════ + # POST-PARSEO: campos comunes a todos los eventos de auth + # ════════════════════════════════════════════════════════════════════════ + + if "_grokparsefailure_auth_generic" not in [tags] { + + # Parsear timestamp syslog → @timestamp + if [auth][timestamp] { + date { + # Syslog omite el año — Logstash asume el año actual + match => [ "[auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + target => "@timestamp" + timezone => "America/Bogota" + tag_on_failure => ["_dateparsefailure_auth"] + } + mutate { remove_field => [ "[auth][timestamp]" ] } + } + + # Campos base + mutate { + add_field => { + "[event][kind]" => "event" + "[event][module]" => "system" + } + convert => { + "[process][pid]" => "integer" + "[source][port]" => "integer" + "[server][port]" => "integer" + } + } + + # Detectar hora de acceso fuera de horario laboral (R8.4 — login fuera de horario) + # Horario laboral: 09:00 – 18:00 GMT-5 + ruby { + code => ' + ts = event.get("@timestamp") + if ts + # Evaluar horario local GMT-5 para login fuera de horario + local_time = ts.time.getlocal("-05:00") + hour = local_time.hour + wday = local_time.wday # 0=domingo, 6=sábado + is_weekend = (wday == 0 || wday == 6) + is_after_hours = (hour < 9 || hour >= 18) + action = event.get("[event][action]") + actions = Array(action).compact + if (is_after_hours || is_weekend) && actions.include?("authentication_success") + event.set("[siem][after_hours_login]", true) + event.tag("after_hours_login") + end + end + ' + } + } + } +} diff --git a/logstash/pipeline/05-filter-syslog.conf b/logstash/pipeline/05-filter-syslog.conf new file mode 100644 index 0000000..6cfb444 --- /dev/null +++ b/logstash/pipeline/05-filter-syslog.conf @@ -0,0 +1,175 @@ +# ============================================================================= +# PIPELINE 05 — FILTRO SYSLOG +# Parsea /var/log/syslog, /var/log/messages, /var/log/kern.log +# Detecta: UFW blocks, OOM killer, errores de disco, fallos de servicio +# ============================================================================= + +filter { + + if [event][dataset] == "system.syslog" { + + # ── Patrón 1: UFW / iptables BLOCK ──────────────────────────────────── + # Apr 25 14:32:01 web-01 kernel: [UFW BLOCK] IN=eth0 OUT= MAC=... SRC=185.220.101.34 DST=10.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=47 ID=12345 DF PROTO=TCP SPT=54321 DPT=22 + if [message] =~ /UFW BLOCK|iptables.*DROP|DROPPED/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} kernel: \[%{DATA:[syslog][kernel_time]}\] \[%{DATA:[firewall][rule]}\] IN=%{DATA:[network][interface][ingress][name]} OUT=%{DATA:[network][interface][egress][name]}(?: MAC=%{DATA:[network][mac]})? SRC=%{IPORHOST:[source][ip]} DST=%{IPORHOST:[destination][ip]}(?: LEN=%{NUMBER:[network][bytes]:int})?(?: TTL=%{NUMBER:[network][ttl]:int})? PROTO=%{WORD:[network][transport]}(?: SPT=%{NUMBER:[source][port]:int})?(?: DPT=%{NUMBER:[destination][port]:int})?" + } + tag_on_failure => ["_grokparsefailure_ufw"] + } + if "_grokparsefailure_ufw" not in [tags] { + mutate { + add_field => { + "[event][category]" => "network" + "[event][type]" => "denied" + "[event][action]" => "firewall_block" + "[event][kind]" => "event" + "[event][outcome]" => "failure" + } + add_tag => ["firewall_block"] + } + } + } + + # ── Patrón 2: OOM Killer ─────────────────────────────────────────────── + # kernel: Out of memory: Kill process 1234 (python3) score 800 or sacrifice child + else if [message] =~ /Out of memory|oom-kill|OOM/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} kernel(?:\[%{POSINT:[process][pid]:int}\])?: (?:Out of memory: Kill process %{NUMBER:[siem][oom_killed_pid]:int} \(%{DATA:[siem][oom_killed_process]}\) score %{NUMBER:[siem][oom_score]:int}|%{GREEDYDATA:[syslog][oom_message]})" + } + tag_on_failure => ["_grokparsefailure_oom"] + } + mutate { + add_field => { + "[event][category]" => "host" + "[event][type]" => "error" + "[event][kind]" => "alert" + "[log][level]" => "critical" + } + add_tag => ["oom_killer", "system_critical"] + } + } + + # ── Patrón 3: Errores de disco / I/O ────────────────────────────────── + # kernel: Buffer I/O error on dev sda1, logical block 123456, lost async page write + # kernel: EXT4-fs error (device sda1): ... + else if [message] =~ /I\/O error|EXT4-fs error|XFS.*error|disk.*error|SCSI error/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} kernel(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYDATA:[error][message]}" + } + tag_on_failure => ["_grokparsefailure_disk"] + } + mutate { + add_field => { + "[event][category]" => "host" + "[event][type]" => "error" + "[event][kind]" => "alert" + "[log][level]" => "error" + } + add_tag => ["disk_error", "system_warning"] + } + } + + # ── Patrón 4: Systemd — fallo de servicio ──────────────────────────── + # systemd[1]: nginx.service: Main process exited, code=killed, status=9/KILL + # systemd[1]: nginx.service: Failed with result 'exit-code'. + else if [message] =~ /systemd.*[Ff]ailed|\.service.*[Ff]ailed|\.service.*error/ { + grok { + match => { + "message" => [ + "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} systemd(?:\[%{POSINT:[process][pid]:int}\])?: %{DATA:[service][name]}\.service: (?:Main process exited, code=%{DATA:[process][exit_code]}, status=%{DATA:[process][exit_status]}|Failed with result '%{DATA:[service][result]}'|%{GREEDYDATA:[service][message]})", + "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} systemd(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYDATA:[service][message]}" + ] + } + tag_on_failure => ["_grokparsefailure_systemd_fail"] + } + mutate { + add_field => { + "[event][category]" => "process" + "[event][type]" => "error" + "[event][kind]" => "event" + "[log][level]" => "error" + } + add_tag => ["service_failure"] + } + } + + # ── Patrón 5: Cron jobs ─────────────────────────────────────────────── + # CRON[1234]: (root) CMD (/usr/bin/python3 /opt/backup.py) + else if [message] =~ /CRON|crond/ { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} CRON\[%{POSINT:[process][pid]:int}\]: \(%{DATA:[user][name]}\) %{DATA:[cron][action]} \(%{GREEDYDATA:[cron][command]}\)" + } + tag_on_failure => ["_grokparsefailure_cron"] + } + mutate { + add_field => { + "[event][category]" => "process" + "[event][type]" => "start" + "[event][kind]" => "event" + } + add_tag => ["cron_job"] + } + } + + # ── Patrón 6: Syslog genérico RFC 3164 ─────────────────────────────── + # Apr 25 14:32:01 web-01 ntpd[1234]: synchronized to 192.168.1.1, stratum 2 + else { + grok { + match => { + "message" => [ + "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYDATA:[syslog][message]}", + "%{SYSLOGTIMESTAMP:[syslog][timestamp]} %{HOSTNAME:[host][name]} %{GREEDYDATA:[syslog][message]}" + ] + } + tag_on_failure => ["_grokparsefailure_syslog_generic"] + } + mutate { + add_field => { + "[event][category]" => "host" + "[event][kind]" => "event" + } + } + } + + # ── POST-PARSEO: campos comunes a todos los syslog ──────────────────── + if "_grokparsefailure_syslog_generic" not in [tags] { + + # Timestamp → @timestamp + if [syslog][timestamp] { + date { + match => [ "[syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + target => "@timestamp" + timezone => "America/Bogota" + tag_on_failure => ["_dateparsefailure_syslog"] + } + mutate { remove_field => ["[syslog][timestamp]"] } + } + + # Limpiar y tipar + mutate { + add_field => { "[event][module]" => "system" } + convert => { "[process][pid]" => "integer" } + remove_field => ["[syslog][kernel_time]"] + } + + # Extraer nivel de log del campo si viene en el mensaje + if [message] and ("error" in [message] or "[error]" in [message]) { + if ![log][level] { + mutate { add_field => { "[log][level]" => "error" } } + } + } else if [message] and ("warn" in [message] or "warning" in [message] or "[warn]" in [message]) { + if ![log][level] { + mutate { add_field => { "[log][level]" => "warning" } } + } + } else if [message] and ("info" in [message] or "notice" in [message] or "[info]" in [message]) { + if ![log][level] { + mutate { add_field => { "[log][level]" => "info" } } + } + } + } + } +} diff --git a/logstash/pipeline/06-enrich.conf b/logstash/pipeline/06-enrich.conf new file mode 100644 index 0000000..a291c25 --- /dev/null +++ b/logstash/pipeline/06-enrich.conf @@ -0,0 +1,154 @@ +# ============================================================================= +# PIPELINE 06 — ENRIQUECIMIENTO GLOBAL +# Se ejecuta sobre TODOS los eventos después de los filtros por fuente +# 1. GeoIP lookup en source.ip +# 2. User-Agent parsing (para logs nginx) +# 3. Mutate final: limpieza, conversión de tipos, eliminación de campos internos +# 4. Marcar logs malformados para enviar al índice de errores +# ============================================================================= + +filter { + + # ════════════════════════════════════════════════════════════════════════════ + # 1. GeoIP — enriquecer source.ip con información geográfica + # ════════════════════════════════════════════════════════════════════════════ + # Solo para IPs públicas (no enriquecer RFC 1918 ni loopback) + if [source][ip] { + # Filtrar IPs privadas / loopback / link-local + if [source][ip] !~ /^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|::1|fe80:)/ { + geoip { + source => "[source][ip]" + target => "[source]" + # Campos que queremos — menos campos = índice más pequeño + fields => [ + "city_name", + "country_code2", + "country_name", + "continent_name", + "region_iso_code", + "region_name", + "location", + "timezone" + ] + tag_on_failure => ["_geoip_lookup_failure"] + } + + # ASN lookup omitido: la ruta MMDB hardcodeada no existe en Logstash 8.13. + } else { + # IP privada — enriquecer con red interna conocida + mutate { + add_field => { + "[source][geo][country_name]" => "Internal Network" + "[source][geo][country_iso_code]" => "LAN" + } + } + } + } + + # GeoIP también en destination.ip si existe + if [destination][ip] { + if [destination][ip] !~ /^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.)/ { + geoip { + source => "[destination][ip]" + target => "[destination]" + fields => ["city_name", "country_code2", "location"] + tag_on_failure => [] + } + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # 2. USER-AGENT parsing + # ════════════════════════════════════════════════════════════════════════════ + if [user_agent][original] and [user_agent][original] != "-" and [user_agent][original] != "" { + useragent { + source => "[user_agent][original]" + target => "[user_agent]" + # Preservar el original después del parse + } + # Restaurar el campo original que useragent puede sobreescribir + if ![user_agent][original] { + mutate { + rename => { "[user_agent][name]" => "[user_agent][original]" } + } + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # 3. MUTATE FINAL — limpieza y normalización de todos los eventos + # ════════════════════════════════════════════════════════════════════════════ + mutate { + # ── Eliminar campos internos de Filebeat/Logstash ────────────────────── + remove_field => [ + "agent.ephemeral_id", + "ecs", # ya mapeado en ecs.version + "[log][offset]", + "input", + "prospector", + "beat", + "[host][containerized]", + "[host][os][codename]" + ] + + # ── Conversión de tipos ──────────────────────────────────────────────── + convert => { + "[http][response][status_code]" => "integer" + "[http][response][body][bytes]" => "integer" + "[source][port]" => "integer" + "[destination][port]" => "integer" + "[process][pid]" => "integer" + "[network][bytes]" => "integer" + "[network][ttl]" => "integer" + } + + # ── Normalizar strings vacíos y guiones a ausencia ───────────────────── + gsub => [ + "[user][name]", "^-$", "", + "[http][request][referrer]", "^-$", "", + "[user_agent][original]", "^-$", "" + ] + } + + # ── Establecer severidad SIEM según tipo de evento ──────────────────────── + ruby { + code => ' + tags = event.get("tags") || [] + if tags.include?("sqli_detected") || tags.include?("path_traversal") || + tags.include?("max_attempts_exceeded") || tags.include?("high_risk_sudo") + event.set("[siem][severity]", "high") + event.set("[event][severity]", 3) + elsif tags.include?("auth_failure") || tags.include?("sensitive_path_access") || + tags.include?("security_scanner") || tags.include?("after_hours_login") || + tags.include?("firewall_block") || tags.include?("xss_detected") + event.set("[siem][severity]", "medium") + event.set("[event][severity]", 2) + elsif tags.include?("auth_success") || tags.include?("sudo_command") || + tags.include?("service_failure") + event.set("[siem][severity]", "low") + event.set("[event][severity]", 1) + else + event.set("[siem][severity]", "informational") + event.set("[event][severity]", 0) + end + ' + } + + # ════════════════════════════════════════════════════════════════════════════ + # 4. IDENTIFICAR LOGS MALFORMADOS → índice de errores + # ════════════════════════════════════════════════════════════════════════════ + # Un log es malformado si tiene cualquier tag de parsefailure + if [tags] { + ruby { + code => ' + tags = event.get("tags") || [] + failure_tags = tags.select { |t| t.to_s.start_with?("_grokparsefailure") } + if failure_tags.length > 0 + event.set("[@metadata][index]", "logs-errors") + event.set("[error][type]", "grok_parse_failure") + event.set("[error][message]", "Failed to parse with patterns: " + failure_tags.join(", ")) + event.set("[error][tags]", failure_tags) + end + ' + } + } +} diff --git a/logstash/pipeline/07-output.conf b/logstash/pipeline/07-output.conf new file mode 100644 index 0000000..e4d1cf9 --- /dev/null +++ b/logstash/pipeline/07-output.conf @@ -0,0 +1,169 @@ +# ============================================================================= +# PIPELINE 07 — OUTPUT +# Routing inteligente a índices Elasticsearch según dataset y resultado de parseo +# Índice de errores separado para logs malformados (R8.3 — tasa parseo > 95%) +# ============================================================================= + +output { + + # ── Configuración TLS compartida (DRY via variables) ────────────────────── + # Usamos un bloque separado por índice para control granular + + # ════════════════════════════════════════════════════════════════════════════ + # ÍNDICE DE ERRORES — logs malformados / no parseados + # Criterio R8.3: rechazar logs malformados a índice separado con motivo + # ════════════════════════════════════════════════════════════════════════════ + if [@metadata][index] == "logs-errors" { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + # Índice con fecha para facilitar purga + index => "logs-errors-%{+YYYY.MM.dd}" + action => "create" + + # Bulk settings — errores son menos frecuentes + pool_max => 1 + + # Template propio para el índice de errores + manage_template => false # ya creado por init-ilm.sh + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-errors" + + # Retry en caso de fallo transitorio + retry_initial_interval => 2 + retry_max_interval => 64 + } + + # También loggear en stderr para monitoreo (opcional) + # stdout { codec => rubydebug } + } + + # ════════════════════════════════════════════════════════════════════════════ + # NGINX — access y error logs + # ════════════════════════════════════════════════════════════════════════════ + else if [@metadata][index] == "logs-nginx" { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + # Subdataset en el nombre del índice (access vs error) + index => "logs-nginx-%{+YYYY.MM.dd}" + action => "create" + + manage_template => false + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-nginx" + + # Bulk settings ajustados para alto volumen de logs web + pool_max => 2 + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # AUTH — logs de autenticación + # ════════════════════════════════════════════════════════════════════════════ + else if [@metadata][index] == "logs-auth" { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + index => "logs-auth-%{+YYYY.MM.dd}" + action => "create" + + manage_template => false + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-auth" + + # Auth logs son críticos — retry agresivo + retry_initial_interval => 1 + retry_max_interval => 32 + pool_max => 2 + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # SYSLOG — logs de sistema + # ════════════════════════════════════════════════════════════════════════════ + else if [@metadata][index] == "logs-syslog" { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + index => "logs-syslog-%{+YYYY.MM.dd}" + action => "create" + + manage_template => false + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-syslog" + + pool_max => 1 + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # KUBERNETES — logs de contenedores + # ════════════════════════════════════════════════════════════════════════════ + else if [@metadata][index] == "logs-k8s" { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + index => "logs-k8s-%{+YYYY.MM.dd}" + action => "create" + + manage_template => false + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-k8s" + + pool_max => 2 + } + } + + # ════════════════════════════════════════════════════════════════════════════ + # FALLBACK — eventos sin índice clasificado + # Van al índice de errores con motivo "unclassified" + # ════════════════════════════════════════════════════════════════════════════ + else { + elasticsearch { + hosts => ["https://es01:9200", "https://es02:9200", "https://es03:9200"] + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + ssl_enabled => true + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca/ca.crt"] + + index => "logs-errors-%{+YYYY.MM.dd}" + action => "create" + + manage_template => false + ilm_enabled => true + ilm_policy => "siem-logs-policy" + ilm_rollover_alias => "logs-errors" + } + + # Loggear en Logstash para debugging (descomentar si se investigan fallbacks) + # stdout { + # codec => rubydebug { metadata => true } + # } + } +} diff --git a/logstash/test-pipeline.sh b/logstash/test-pipeline.sh new file mode 100644 index 0000000..5297752 --- /dev/null +++ b/logstash/test-pipeline.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# ── Probar el pipeline con líneas de muestra ────────────────────────────────── +# Uso: bash logstash/test-pipeline.sh + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CACERT="${CACERT:-$ROOT_DIR/setup/certs/ca/ca.crt}" + +echo "=== Test de patrones grok contra líneas de muestra ===" + +# Líneas de prueba +NGINX_LINE='185.220.101.34 - - [25/Apr/2024:14:32:01 +0000] "GET /api/v1/users?id=1 HTTP/1.1" 200 1842 "-" "curl/7.68.0"' +AUTH_FAIL='Apr 25 14:32:01 web-01 sshd[12345]: Failed password for root from 185.220.101.34 port 54321 ssh2' +AUTH_OK='Apr 25 14:32:01 web-01 sshd[12345]: Accepted publickey for ubuntu from 192.168.1.20 port 52341 ssh2' +SYSLOG_UFW='Apr 25 14:32:01 web-01 kernel: [123456.789] [UFW BLOCK] IN=eth0 OUT= SRC=185.220.101.34 DST=10.0.0.1 LEN=60 TTL=47 PROTO=TCP SPT=54321 DPT=22' +SQLI_LINE='185.220.101.34 - - [25/Apr/2024:14:32:01 +0000] "GET /api?id=1'"'"' OR '"'"'1'"'"'='"'"'1 HTTP/1.1" 200 512 "-" "sqlmap/1.7.2"' + +echo "" +echo "Verificando que Logstash esté levantado..." +curl -s http://localhost:9600/ | python3 -m json.tool | grep -E '"status"|"version"' || echo "ERROR: Logstash no responde en :9600" + +echo "" +echo "=== Estadísticas del pipeline ===" +curl -s http://localhost:9600/_node/stats/pipelines/main | python3 -c " +import json, sys +d = json.load(sys.stdin) +p = d.get('pipelines', {}).get('main', {}).get('events', {}) +print(f' Eventos in: {p.get(\"in\", 0):,}') +print(f' Eventos out: {p.get(\"out\", 0):,}') +print(f' Eventos filtrados:{p.get(\"filtered\", 0):,}') +print(f' Duración (ms): {p.get(\"duration_in_millis\", 0):,}') +" 2>/dev/null || echo " No se pudo obtener estadísticas" + +echo "" +echo "=== Verificar índices creados ===" +curl -s --cacert "$CACERT" \ + -u "elastic:${ELASTIC_PASSWORD:-SiemElastic2026!}" \ + "https://localhost:9200/_cat/indices/logs-*?v&s=index" 2>/dev/null || \ + echo " No se puede conectar a ES (ejecutar desde el host con el stack levantado)" + +echo "" +echo "=== Tasa de parseo por índice ===" +for idx in logs-nginx logs-auth logs-syslog logs-k8s logs-errors; do + COUNT=$(curl -s --cacert "$CACERT" \ + -u "elastic:${ELASTIC_PASSWORD:-SiemElastic2026!}" \ + "https://localhost:9200/${idx}-*/_count" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin).get('count',0))" 2>/dev/null || echo 0) + printf " %-20s %s documentos\n" "${idx}-*:" "$COUNT" +done diff --git a/rules/all-rules.ndjson b/rules/all-rules.ndjson new file mode 100644 index 0000000..8ac46ea --- /dev/null +++ b/rules/all-rules.ndjson @@ -0,0 +1,5 @@ +{"name": "SIEM - Brute Force SSH Detectado", "description": "Detecta intentos de brute force SSH: 5 o más fallos de autenticación desde la misma IP en 60 segundos. Cubre usuarios válidos e inválidos. Severidad alta — responder según playbook docs/playbooks/brute-force-response.md", "risk_score": 85, "severity": "high", "enabled": true, "type": "threshold", "query": "event.dataset:\"system.auth\" AND event.outcome:\"failure\" AND (event.action:\"authentication_failure\" OR tags:\"auth_failure\") AND process.name:\"sshd\"", "language": "kuery", "index": ["logs-auth*"], "threshold": {"field": ["source.ip.keyword"], "value": 5, "cardinality": []}, "from": "now-60s", "interval": "1m", "max_signals": 100, "tags": ["SIEM", "Brute Force", "SSH", "Authentication", "T1110", "Credential Access"], "threat": [{"framework": "MITRE ATT&CK", "tactic": {"id": "TA0006", "name": "Credential Access", "reference": "https://attack.mitre.org/tactics/TA0006/"}, "technique": [{"id": "T1110", "name": "Brute Force", "reference": "https://attack.mitre.org/techniques/T1110/", "subtechnique": [{"id": "T1110.001", "name": "Password Guessing", "reference": "https://attack.mitre.org/techniques/T1110/001/"}]}]}], "actions": [], "references": ["https://www.elastic.co/guide/en/security/current/prebuilt-rule-8-2-1-potential-ssh-brute-force-detected.html"], "note": "## Investigación\n1. Verificar en Kibana → Security → Alerts la IP origen (`source.ip`)\n2. Revisar si la IP está en listas de threat intel conocidas\n3. Ejecutar: `GET logs-auth*/_search` filtrando por esa IP en los últimos 5 minutos\n4. Verificar si algún intento tuvo `event.outcome:success` (brute force exitoso)\n\n## Contención inmediata\n- Bloquear IP en UFW: `sudo ufw deny from to any`\n- Si hubo éxito: revocar sesión activa y cambiar credenciales\n\n## Falsos positivos esperados\n- Sistemas de monitoreo legítimos con autenticación fallida\n- Usuarios que olvidaron credenciales (menos de 10 intentos, IP interna)", "false_positives": ["Sistemas de monitoreo o healthcheck con credenciales desactualizadas", "Usuarios internos con múltiples intentos fallidos por error de teclado"], "author": ["Equipo SIEM - Proyecto 8"], "license": "Elastic License v2", "version": 1, "rule_id": "siem-rule-brute-force-ssh"} +{"name": "SIEM - Login SSH Exitoso Fuera de Horario Laboral", "description": "Detecta autenticaciones SSH exitosas fuera del horario laboral definido (lunes-viernes 09:00-18:00 GMT-5) o durante fines de semana. El pipeline de Logstash marca estos eventos con el campo siem.after_hours_login=true. Puede indicar acceso no autorizado, cuenta comprometida o trabajo remoto no declarado.", "risk_score": 63, "severity": "medium", "enabled": true, "type": "query", "query": "event.dataset:\"system.auth\" AND event.outcome:\"success\" AND event.action:\"authentication_success\" AND siem.after_hours_login:true AND process.name:\"sshd\"", "language": "kuery", "index": ["logs-auth*"], "from": "now-2h", "interval": "5m", "max_signals": 100, "tags": ["SIEM", "After Hours", "Authentication", "Insider Threat", "T1078", "Valid Accounts"], "threat": [{"framework": "MITRE ATT&CK", "tactic": {"id": "TA0001", "name": "Initial Access", "reference": "https://attack.mitre.org/tactics/TA0001/"}, "technique": [{"id": "T1078", "name": "Valid Accounts", "reference": "https://attack.mitre.org/techniques/T1078/", "subtechnique": [{"id": "T1078.003", "name": "Local Accounts", "reference": "https://attack.mitre.org/techniques/T1078/003/"}]}]}, {"framework": "MITRE ATT&CK", "tactic": {"id": "TA0003", "name": "Persistence", "reference": "https://attack.mitre.org/tactics/TA0003/"}, "technique": [{"id": "T1078", "name": "Valid Accounts", "reference": "https://attack.mitre.org/techniques/T1078/"}]}], "actions": [], "note": "## Investigación\n1. Identificar `user.name` y `source.ip` — ¿es un empleado conocido?\n2. Verificar `source.geo.country_iso_code` — ¿acceso desde país inusual?\n3. Revisar actividad post-login: comandos sudo, accesos a archivos sensibles\n4. Consultar con el usuario si el acceso fue intencional\n\n## Contención (si sospechoso)\n- Bloquear sesión activa: `sudo pkill -u `\n- Cambiar contraseña o revocar clave SSH\n- Habilitar MFA si no está activo\n\n## Ajuste de horario\nModificar en 04-filter-auth.conf la condición Ruby:\n `hour < 9 || hour >= 18` → ajustar según zona horaria del equipo\n `wday == 0 || wday == 6` → fines de semana (0=domingo, 6=sábado)\n\n## Falsos positivos esperados\n- Empleados con trabajo flexible o en otras zonas horarias\n- Sistemas de CI/CD que se autentican automáticamente", "false_positives": ["Empleados con horario flexible o en zonas horarias diferentes (UTC-5 para Colombia)", "Pipelines de CI/CD o sistemas automatizados que corren en horarios no laborales", "Administradores de guardia con acceso nocturno autorizado"], "author": ["Equipo SIEM - Proyecto 8"], "license": "Elastic License v2", "version": 1, "rule_id": "siem-rule-login-after-hours"} +{"name": "SIEM - Flood de Errores HTTP 404 (Enumeración de Recursos)", "description": "Detecta enumeración de recursos web: una IP genera 15 o más respuestas 404 en 60 segundos sobre rutas distintas. Patrón típico de herramientas como dirbuster, gobuster, wfuzz o nikto buscando archivos/directorios ocultos.", "risk_score": 47, "severity": "medium", "enabled": true, "type": "threshold", "query": "event.dataset:\"nginx.access\" AND http.response.status_code:404", "language": "kuery", "index": ["logs-nginx*"], "threshold": {"field": ["source.ip.keyword"], "value": 15, "cardinality": [{"field": "url.path.keyword", "value": 10}]}, "from": "now-60s", "interval": "1m", "max_signals": 100, "tags": ["SIEM", "Web Enumeration", "404 Flood", "T1595", "Reconnaissance"], "threat": [{"framework": "MITRE ATT&CK", "tactic": {"id": "TA0043", "name": "Reconnaissance", "reference": "https://attack.mitre.org/tactics/TA0043/"}, "technique": [{"id": "T1595", "name": "Active Scanning", "reference": "https://attack.mitre.org/techniques/T1595/"}]}, {"framework": "MITRE ATT&CK", "tactic": {"id": "TA0007", "name": "Discovery", "reference": "https://attack.mitre.org/tactics/TA0007/"}, "technique": [{"id": "T1083", "name": "File and Directory Discovery", "reference": "https://attack.mitre.org/techniques/T1083/"}]}], "actions": [], "note": "## Investigación\n1. Identificar `source.ip` y revisar `url.path` de los 404 — ¿busca WordPress, phpMyAdmin, .env?\n2. Revisar `user_agent.original` — herramientas como gobuster/dirbuster tienen UAs característicos\n3. Verificar si algún path devolvió 200 en la misma ventana temporal (éxito del enumerador)\n4. Revisar logs-auth* — ¿el mismo IP intentó autenticación tras encontrar algo?\n\n## Contención\n- Rate-limit desde esa IP en nginx: `limit_req_zone`\n- Bloquear si user-agent es un scanner conocido\n\n## Falsos positivos esperados\n- Links rotos en aplicación (muchos 404 legítimos desde diferentes usuarios)\n- Bots de crawling como Googlebot con paths inexistentes", "false_positives": ["Aplicaciones con muchos enlaces rotos que generan 404s masivos", "Crawlers de buscadores (Googlebot, Bingbot) explorando rutas inexistentes", "Pruebas de integración o smoke tests que golpean rutas no existentes"], "author": ["Equipo SIEM - Proyecto 8"], "license": "Elastic License v2", "version": 1, "rule_id": "siem-rule-multiple-404"} +{"name": "SIEM - Escaneo de Puertos Detectado", "description": "Detecta escaneos de puertos: una misma IP origen contacta 20 o más puertos de destino distintos en 30 segundos. Cubre nmap SYN scan, connect scan y escaneos UDP. Alta cardinalidad de puertos destino es indicador fuerte de reconocimiento.", "risk_score": 73, "severity": "medium", "enabled": true, "type": "threshold", "query": "event.category:\"network\" AND event.action:\"firewall_block\" AND tags:\"firewall_block\" AND network.transport:(\"tcp\" OR \"udp\" OR \"TCP\" OR \"UDP\")", "language": "kuery", "index": ["logs-syslog*"], "threshold": {"field": ["source.ip.keyword"], "value": 20, "cardinality": [{"field": "destination.port", "value": 20}]}, "from": "now-30s", "interval": "1m", "max_signals": 100, "tags": ["SIEM", "Port Scan", "Reconnaissance", "T1046", "Discovery"], "threat": [{"framework": "MITRE ATT&CK", "tactic": {"id": "TA0007", "name": "Discovery", "reference": "https://attack.mitre.org/tactics/TA0007/"}, "technique": [{"id": "T1046", "name": "Network Service Discovery", "reference": "https://attack.mitre.org/techniques/T1046/"}]}, {"framework": "MITRE ATT&CK", "tactic": {"id": "TA0043", "name": "Reconnaissance", "reference": "https://attack.mitre.org/tactics/TA0043/"}, "technique": [{"id": "T1595", "name": "Active Scanning", "reference": "https://attack.mitre.org/techniques/T1595/", "subtechnique": [{"id": "T1595.001", "name": "Scanning IP Blocks", "reference": "https://attack.mitre.org/techniques/T1595/001/"}]}]}], "actions": [], "note": "## Investigación\n1. Identificar `source.ip` y `destination.ip` en la alerta\n2. Revisar `destination.port` para entender qué servicios está buscando\n3. Correlacionar con logs-auth* — ¿intentó acceder a algún servicio abierto?\n4. Verificar si hay intentos de conexión exitosa tras el escaneo\n\n## Contención\n- Bloquear IP inmediatamente en firewall perimetral\n- Revisar si la IP pertenece a un escáner interno autorizado (Nessus, OpenVAS)\n\n## Falsos positivos esperados\n- Escáneres de seguridad internos programados (Nessus, Qualys)\n- Healthchecks de balanceadores de carga que prueban múltiples puertos", "false_positives": ["Escáneres de vulnerabilidades internos autorizados (Nessus, OpenVAS, Qualys)", "Monitoreo de disponibilidad que verifica múltiples puertos de servicio"], "author": ["Equipo SIEM - Proyecto 8"], "license": "Elastic License v2", "version": 1, "rule_id": "siem-rule-port-scan"} +{"name": "SIEM - Acceso a Rutas Web Sensibles", "description": "Detecta peticiones HTTP a rutas críticas de la aplicación: paneles de administración, archivos de configuración, credenciales, herramientas de base de datos y endpoints internos. El pipeline de Logstash enriquece estos eventos con siem.sensitive_path=true. Cualquier acceso — exitoso o fallido — debe investigarse.", "risk_score": 73, "severity": "high", "enabled": true, "type": "query", "query": "event.dataset:\"nginx.access\" AND siem.sensitive_path:true", "language": "kuery", "index": ["logs-nginx*"], "from": "now-5m", "interval": "1m", "max_signals": 200, "tags": ["SIEM", "Sensitive Paths", "Web Attack", "T1190", "Initial Access"], "threat": [{"framework": "MITRE ATT&CK", "tactic": {"id": "TA0001", "name": "Initial Access", "reference": "https://attack.mitre.org/tactics/TA0001/"}, "technique": [{"id": "T1190", "name": "Exploit Public-Facing Application", "reference": "https://attack.mitre.org/techniques/T1190/"}]}, {"framework": "MITRE ATT&CK", "tactic": {"id": "TA0006", "name": "Credential Access", "reference": "https://attack.mitre.org/tactics/TA0006/"}, "technique": [{"id": "T1552", "name": "Unsecured Credentials", "reference": "https://attack.mitre.org/techniques/T1552/", "subtechnique": [{"id": "T1552.001", "name": "Credentials In Files", "reference": "https://attack.mitre.org/techniques/T1552/001/"}]}]}], "actions": [], "note": "## Rutas monitoreadas\n/admin, /wp-admin, /phpmyadmin, /.env, /config, /backup, /.git, /etc/passwd, /proc/, /server-status, /actuator, /api/admin\n\n## Investigación\n1. Revisar `http.response.status_code` — un 200 es crítico (acceso exitoso)\n2. Revisar `source.ip` y `source.geo` — IP externa accediendo a /admin es señal de alarma\n3. Verificar `user_agent.original` — sqlmap, nikto, curl indican herramienta automatizada\n4. Si status 200 en /.env o /config: asumir credenciales comprometidas y rotar inmediatamente\n\n## Contención por severidad\n- Status 200 en ruta crítica → respuesta de incidente inmediata\n- Status 403/404 desde IP externa → monitorear y posiblemente bloquear\n- Status 403 desde IP interna → investigar usuario\n\n## Agregar rutas\nModificar regex en 03-filter-nginx.conf bloque 'Detección de rutas sensibles'", "false_positives": ["Administradores accediendo a /admin desde IPs internas conocidas", "Health checks de Kubernetes que consultan /actuator/health", "Scans de seguridad internos programados"], "author": ["Equipo SIEM - Proyecto 8"], "license": "Elastic License v2", "version": 1, "rule_id": "siem-rule-sensitive-paths"} diff --git a/rules/brute-force-ssh.json b/rules/brute-force-ssh.json new file mode 100644 index 0000000..a4869ae --- /dev/null +++ b/rules/brute-force-ssh.json @@ -0,0 +1,64 @@ +{ + "id": "siem-rule-brute-force-ssh", + "name": "SIEM - Brute Force SSH Detectado", + "description": "Detecta intentos de brute force SSH: 5 o más fallos de autenticación desde la misma IP en 60 segundos. Cubre usuarios válidos e inválidos. Severidad alta — responder según playbook docs/playbooks/brute-force-response.md", + "risk_score": 85, + "severity": "high", + "enabled": true, + "type": "threshold", + "query": "event.dataset:\"system.auth\" AND event.outcome:\"failure\" AND (event.action:\"authentication_failure\" OR tags:\"auth_failure\") AND process.name:\"sshd\"", + "language": "kuery", + "index": ["logs-auth*"], + "threshold": { + "field": ["source.ip.keyword"], + "value": 5, + "cardinality": [] + }, + "from": "now-60s", + "interval": "1m", + "max_signals": 100, + "tags": [ + "SIEM", + "Brute Force", + "SSH", + "Authentication", + "T1110", + "Credential Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0006", + "name": "Credential Access", + "reference": "https://attack.mitre.org/tactics/TA0006/" + }, + "technique": [ + { + "id": "T1110", + "name": "Brute Force", + "reference": "https://attack.mitre.org/techniques/T1110/", + "subtechnique": [ + { + "id": "T1110.001", + "name": "Password Guessing", + "reference": "https://attack.mitre.org/techniques/T1110/001/" + } + ] + } + ] + } + ], + "actions": [], + "references": [ + "https://www.elastic.co/guide/en/security/current/prebuilt-rule-8-2-1-potential-ssh-brute-force-detected.html" + ], + "note": "## Investigación\n1. Verificar en Kibana → Security → Alerts la IP origen (`source.ip`)\n2. Revisar si la IP está en listas de threat intel conocidas\n3. Ejecutar: `GET logs-auth*/_search` filtrando por esa IP en los últimos 5 minutos\n4. Verificar si algún intento tuvo `event.outcome:success` (brute force exitoso)\n\n## Contención inmediata\n- Bloquear IP en UFW: `sudo ufw deny from to any`\n- Si hubo éxito: revocar sesión activa y cambiar credenciales\n\n## Falsos positivos esperados\n- Sistemas de monitoreo legítimos con autenticación fallida\n- Usuarios que olvidaron credenciales (menos de 10 intentos, IP interna)", + "false_positives": [ + "Sistemas de monitoreo o healthcheck con credenciales desactualizadas", + "Usuarios internos con múltiples intentos fallidos por error de teclado" + ], + "author": ["Equipo SIEM - Proyecto 8"], + "license": "Elastic License v2", + "version": 1 +} diff --git a/rules/login-after-hours.json b/rules/login-after-hours.json new file mode 100644 index 0000000..40d6b5e --- /dev/null +++ b/rules/login-after-hours.json @@ -0,0 +1,72 @@ +{ + "id": "siem-rule-login-after-hours", + "name": "SIEM - Login SSH Exitoso Fuera de Horario Laboral", + "description": "Detecta autenticaciones SSH exitosas fuera del horario laboral definido (lunes-viernes 09:00-18:00 GMT-5) o durante fines de semana. El pipeline de Logstash marca estos eventos con el campo siem.after_hours_login=true. Puede indicar acceso no autorizado, cuenta comprometida o trabajo remoto no declarado.", + "risk_score": 63, + "severity": "medium", + "enabled": true, + "type": "query", + "query": "event.dataset:\"system.auth\" AND event.outcome:\"success\" AND event.action:\"authentication_success\" AND siem.after_hours_login:true AND process.name:\"sshd\"", + "language": "kuery", + "index": ["logs-auth*"], + "from": "now-2h", + "interval": "5m", + "max_signals": 100, + "tags": [ + "SIEM", + "After Hours", + "Authentication", + "Insider Threat", + "T1078", + "Valid Accounts" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0001", + "name": "Initial Access", + "reference": "https://attack.mitre.org/tactics/TA0001/" + }, + "technique": [ + { + "id": "T1078", + "name": "Valid Accounts", + "reference": "https://attack.mitre.org/techniques/T1078/", + "subtechnique": [ + { + "id": "T1078.003", + "name": "Local Accounts", + "reference": "https://attack.mitre.org/techniques/T1078/003/" + } + ] + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0003", + "name": "Persistence", + "reference": "https://attack.mitre.org/tactics/TA0003/" + }, + "technique": [ + { + "id": "T1078", + "name": "Valid Accounts", + "reference": "https://attack.mitre.org/techniques/T1078/" + } + ] + } + ], + "actions": [], + "note": "## Investigación\n1. Identificar `user.name` y `source.ip` — ¿es un empleado conocido?\n2. Verificar `source.geo.country_iso_code` — ¿acceso desde país inusual?\n3. Revisar actividad post-login: comandos sudo, accesos a archivos sensibles\n4. Consultar con el usuario si el acceso fue intencional\n\n## Contención (si sospechoso)\n- Bloquear sesión activa: `sudo pkill -u `\n- Cambiar contraseña o revocar clave SSH\n- Habilitar MFA si no está activo\n\n## Ajuste de horario\nModificar en 04-filter-auth.conf la condición Ruby:\n `hour < 9 || hour >= 18` → ajustar según zona horaria del equipo\n `wday == 0 || wday == 6` → fines de semana (0=domingo, 6=sábado)\n\n## Falsos positivos esperados\n- Empleados con trabajo flexible o en otras zonas horarias\n- Sistemas de CI/CD que se autentican automáticamente", + "false_positives": [ + "Empleados con horario flexible o en zonas horarias diferentes (UTC-5 para Colombia)", + "Pipelines de CI/CD o sistemas automatizados que corren en horarios no laborales", + "Administradores de guardia con acceso nocturno autorizado" + ], + "author": ["Equipo SIEM - Proyecto 8"], + "license": "Elastic License v2", + "version": 1 +} diff --git a/rules/multiple-404.json b/rules/multiple-404.json new file mode 100644 index 0000000..7b88767 --- /dev/null +++ b/rules/multiple-404.json @@ -0,0 +1,74 @@ +{ + "id": "siem-rule-multiple-404", + "name": "SIEM - Flood de Errores HTTP 404 (Enumeración de Recursos)", + "description": "Detecta enumeración de recursos web: una IP genera 15 o más respuestas 404 en 60 segundos sobre rutas distintas. Patrón típico de herramientas como dirbuster, gobuster, wfuzz o nikto buscando archivos/directorios ocultos.", + "risk_score": 47, + "severity": "medium", + "enabled": true, + "type": "threshold", + "query": "event.dataset:\"nginx.access\" AND http.response.status_code:404", + "language": "kuery", + "index": ["logs-nginx*"], + "threshold": { + "field": ["source.ip.keyword"], + "value": 15, + "cardinality": [ + { + "field": "url.path.keyword", + "value": 10 + } + ] + }, + "from": "now-60s", + "interval": "1m", + "max_signals": 100, + "tags": [ + "SIEM", + "Web Enumeration", + "404 Flood", + "T1595", + "Reconnaissance" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0043", + "name": "Reconnaissance", + "reference": "https://attack.mitre.org/tactics/TA0043/" + }, + "technique": [ + { + "id": "T1595", + "name": "Active Scanning", + "reference": "https://attack.mitre.org/techniques/T1595/" + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0007", + "name": "Discovery", + "reference": "https://attack.mitre.org/tactics/TA0007/" + }, + "technique": [ + { + "id": "T1083", + "name": "File and Directory Discovery", + "reference": "https://attack.mitre.org/techniques/T1083/" + } + ] + } + ], + "actions": [], + "note": "## Investigación\n1. Identificar `source.ip` y revisar `url.path` de los 404 — ¿busca WordPress, phpMyAdmin, .env?\n2. Revisar `user_agent.original` — herramientas como gobuster/dirbuster tienen UAs característicos\n3. Verificar si algún path devolvió 200 en la misma ventana temporal (éxito del enumerador)\n4. Revisar logs-auth* — ¿el mismo IP intentó autenticación tras encontrar algo?\n\n## Contención\n- Rate-limit desde esa IP en nginx: `limit_req_zone`\n- Bloquear si user-agent es un scanner conocido\n\n## Falsos positivos esperados\n- Links rotos en aplicación (muchos 404 legítimos desde diferentes usuarios)\n- Bots de crawling como Googlebot con paths inexistentes", + "false_positives": [ + "Aplicaciones con muchos enlaces rotos que generan 404s masivos", + "Crawlers de buscadores (Googlebot, Bingbot) explorando rutas inexistentes", + "Pruebas de integración o smoke tests que golpean rutas no existentes" + ], + "author": ["Equipo SIEM - Proyecto 8"], + "license": "Elastic License v2", + "version": 1 +} diff --git a/rules/port-scan.json b/rules/port-scan.json new file mode 100644 index 0000000..3feb9e6 --- /dev/null +++ b/rules/port-scan.json @@ -0,0 +1,80 @@ +{ + "id": "siem-rule-port-scan", + "name": "SIEM - Escaneo de Puertos Detectado", + "description": "Detecta escaneos de puertos: una misma IP origen contacta 20 o más puertos de destino distintos en 30 segundos. Cubre nmap SYN scan, connect scan y escaneos UDP. Alta cardinalidad de puertos destino es indicador fuerte de reconocimiento.", + "risk_score": 73, + "severity": "medium", + "enabled": true, + "type": "threshold", + "query": "event.category:\"network\" AND event.action:\"firewall_block\" AND tags:\"firewall_block\" AND network.transport:(\"tcp\" OR \"udp\" OR \"TCP\" OR \"UDP\")", + "language": "kuery", + "index": ["logs-syslog*"], + "threshold": { + "field": ["source.ip.keyword"], + "value": 20, + "cardinality": [ + { + "field": "destination.port", + "value": 20 + } + ] + }, + "from": "now-30s", + "interval": "1m", + "max_signals": 100, + "tags": [ + "SIEM", + "Port Scan", + "Reconnaissance", + "T1046", + "Discovery" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0007", + "name": "Discovery", + "reference": "https://attack.mitre.org/tactics/TA0007/" + }, + "technique": [ + { + "id": "T1046", + "name": "Network Service Discovery", + "reference": "https://attack.mitre.org/techniques/T1046/" + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0043", + "name": "Reconnaissance", + "reference": "https://attack.mitre.org/tactics/TA0043/" + }, + "technique": [ + { + "id": "T1595", + "name": "Active Scanning", + "reference": "https://attack.mitre.org/techniques/T1595/", + "subtechnique": [ + { + "id": "T1595.001", + "name": "Scanning IP Blocks", + "reference": "https://attack.mitre.org/techniques/T1595/001/" + } + ] + } + ] + } + ], + "actions": [], + "note": "## Investigación\n1. Identificar `source.ip` y `destination.ip` en la alerta\n2. Revisar `destination.port` para entender qué servicios está buscando\n3. Correlacionar con logs-auth* — ¿intentó acceder a algún servicio abierto?\n4. Verificar si hay intentos de conexión exitosa tras el escaneo\n\n## Contención\n- Bloquear IP inmediatamente en firewall perimetral\n- Revisar si la IP pertenece a un escáner interno autorizado (Nessus, OpenVAS)\n\n## Falsos positivos esperados\n- Escáneres de seguridad internos programados (Nessus, Qualys)\n- Healthchecks de balanceadores de carga que prueban múltiples puertos", + "false_positives": [ + "Escáneres de vulnerabilidades internos autorizados (Nessus, OpenVAS, Qualys)", + "Monitoreo de disponibilidad que verifica múltiples puertos de servicio" + ], + "author": ["Equipo SIEM - Proyecto 8"], + "license": "Elastic License v2", + "version": 1 +} diff --git a/rules/sensitive-paths.json b/rules/sensitive-paths.json new file mode 100644 index 0000000..f8a97b9 --- /dev/null +++ b/rules/sensitive-paths.json @@ -0,0 +1,71 @@ +{ + "id": "siem-rule-sensitive-paths", + "name": "SIEM - Acceso a Rutas Web Sensibles", + "description": "Detecta peticiones HTTP a rutas críticas de la aplicación: paneles de administración, archivos de configuración, credenciales, herramientas de base de datos y endpoints internos. El pipeline de Logstash enriquece estos eventos con siem.sensitive_path=true. Cualquier acceso — exitoso o fallido — debe investigarse.", + "risk_score": 73, + "severity": "high", + "enabled": true, + "type": "query", + "query": "event.dataset:\"nginx.access\" AND siem.sensitive_path:true", + "language": "kuery", + "index": ["logs-nginx*"], + "from": "now-5m", + "interval": "1m", + "max_signals": 200, + "tags": [ + "SIEM", + "Sensitive Paths", + "Web Attack", + "T1190", + "Initial Access" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0001", + "name": "Initial Access", + "reference": "https://attack.mitre.org/tactics/TA0001/" + }, + "technique": [ + { + "id": "T1190", + "name": "Exploit Public-Facing Application", + "reference": "https://attack.mitre.org/techniques/T1190/" + } + ] + }, + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0006", + "name": "Credential Access", + "reference": "https://attack.mitre.org/tactics/TA0006/" + }, + "technique": [ + { + "id": "T1552", + "name": "Unsecured Credentials", + "reference": "https://attack.mitre.org/techniques/T1552/", + "subtechnique": [ + { + "id": "T1552.001", + "name": "Credentials In Files", + "reference": "https://attack.mitre.org/techniques/T1552/001/" + } + ] + } + ] + } + ], + "actions": [], + "note": "## Rutas monitoreadas\n/admin, /wp-admin, /phpmyadmin, /.env, /config, /backup, /.git, /etc/passwd, /proc/, /server-status, /actuator, /api/admin\n\n## Investigación\n1. Revisar `http.response.status_code` — un 200 es crítico (acceso exitoso)\n2. Revisar `source.ip` y `source.geo` — IP externa accediendo a /admin es señal de alarma\n3. Verificar `user_agent.original` — sqlmap, nikto, curl indican herramienta automatizada\n4. Si status 200 en /.env o /config: asumir credenciales comprometidas y rotar inmediatamente\n\n## Contención por severidad\n- Status 200 en ruta crítica → respuesta de incidente inmediata\n- Status 403/404 desde IP externa → monitorear y posiblemente bloquear\n- Status 403 desde IP interna → investigar usuario\n\n## Agregar rutas\nModificar regex en 03-filter-nginx.conf bloque 'Detección de rutas sensibles'", + "false_positives": [ + "Administradores accediendo a /admin desde IPs internas conocidas", + "Health checks de Kubernetes que consultan /actuator/health", + "Scans de seguridad internos programados" + ], + "author": ["Equipo SIEM - Proyecto 8"], + "license": "Elastic License v2", + "version": 1 +} diff --git a/setup/add-dashboard-panels.py b/setup/add-dashboard-panels.py new file mode 100644 index 0000000..61a0c60 --- /dev/null +++ b/setup/add-dashboard-panels.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +add-dashboard-panels.py +Agrega los paneles de visualizaciones a los dashboards después de la importación. +""" + +import json +import time +import sys +import os +from pathlib import Path + +try: + import requests + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + print("ERROR: pip install requests") + sys.exit(1) + +KIBANA_URL = os.getenv("KIBANA_HOST", "https://localhost:5601") +ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic") +ELASTIC_PASS = os.getenv("ELASTIC_PASSWORD","SiemElastic2026!") +CACERT = os.getenv("CACERT", "./setup/certs/ca/ca.crt") + +def kibana_session(): + s = requests.Session() + s.auth = (ELASTIC_USER, ELASTIC_PASS) + s.headers = {"kbn-xsrf": "true"} + s.verify = CACERT if Path(CACERT).exists() else False + return s + +def wait_for_kibana(timeout=180): + print("Esperando a que Kibana esté disponible...") + end_time = time.time() + timeout + while time.time() < end_time: + try: + s = kibana_session() + r = s.get(f"{KIBANA_URL}/api/status", timeout=5) + if r.status_code == 200: + print(" ✓ Kibana disponible") + return True + except: + pass + time.sleep(2) + print(f" ✗ Kibana no respondió") + return False + +def add_panels_to_dashboard(dashboard_id, panels): + """Agrega paneles a un dashboard existente""" + session = kibana_session() + + # Obtener dashboard actual + r = session.get(f"{KIBANA_URL}/api/saved_objects/dashboard/{dashboard_id}") + if r.status_code != 200: + print(f" ✗ No se pudo obtener dashboard {dashboard_id}") + return False + + dashboard = r.json() + if "error" in dashboard: + print(f" ✗ Error: {dashboard['error']['message']}") + return False + + # Actualizar con paneles + dashboard['attributes']['panels'] = panels + + r = session.put( + f"{KIBANA_URL}/api/saved_objects/dashboard/{dashboard_id}", + json={"attributes": dashboard["attributes"]}, + timeout=10 + ) + + if r.status_code not in [200, 201]: + print(f" ✗ Error {r.status_code} actualizando dashboard") + return False + + return True + +def main(): + if not wait_for_kibana(): + sys.exit(1) + + print("\n=== Agregando Paneles a Dashboards ===\n") + + # Paneles para dashboard ejecutivo + exec_panels = [ + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 0, "w": 24, "h": 12}, + "type": "visualization", + "id": "viz-system-health", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 24, "y": 0, "w": 24, "h": 12}, + "type": "visualization", + "id": "viz-top-threats", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 12, "w": 48, "h": 12}, + "type": "visualization", + "id": "viz-alert-trend", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 24, "w": 48, "h": 15}, + "type": "visualization", + "id": "viz-top-ips", + "embeddableConfig": {} + } + ] + + # Paneles para dashboard operacional + ops_panels = [ + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 0, "w": 24, "h": 15}, + "type": "visualization", + "id": "viz-top-users", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 24, "y": 0, "w": 24, "h": 15}, + "type": "visualization", + "id": "viz-top-ips", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 15, "w": 48, "h": 18}, + "type": "visualization", + "id": "viz-alerts-table", + "embeddableConfig": {} + }, + { + "version": "8.19.14", + "gridData": {"x": 0, "y": 33, "w": 48, "h": 20}, + "type": "visualization", + "id": "viz-events-table", + "embeddableConfig": {} + } + ] + + # Agregar paneles + if add_panels_to_dashboard("executive-security-overview", exec_panels): + print(" ✓ Paneles agregados a Executive Dashboard") + + if add_panels_to_dashboard("operational-triage-console", ops_panels): + print(" ✓ Paneles agregados a Operational Dashboard") + + print("\n✓ Dashboards configurados completamente") + +if __name__ == "__main__": + main() diff --git a/setup/check-cluster-health.py b/setup/check-cluster-health.py new file mode 100644 index 0000000..6e7d641 --- /dev/null +++ b/setup/check-cluster-health.py @@ -0,0 +1,88 @@ +import requests +import os +import warnings +from datetime import datetime + +warnings.filterwarnings("ignore", message="Unverified HTTPS request") + +def get_env_var(var_name, default_value): + try: + with open(".env", "r") as f: + for line in f: + if line.startswith(var_name): + return line.split("=")[1].strip().replace('"', '').replace("'", "") + except: + pass + return os.getenv(var_name, default_value) + +ELASTIC_PASSWORD = get_env_var('ELASTIC_PASSWORD', 'SiemElastic2024!') +ELASTIC_HOST = "https://es01:9200" + +def run_diagnostic(): + print("="*70) + print(f"SIEM MASTER DIAGNOSTIC - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("="*70) + + # --- 1. ELASTICSEARCH (R8.1) --- + print("\n[1/4] ELASTICSEARCH CLUSTER") + try: + health = requests.get(f"{ELASTIC_HOST}/_cluster/health", auth=('elastic', ELASTIC_PASSWORD), verify=False, timeout=5).json() + status = health['status'] + color = "OK" if status == "green" else "WARN" if status == "yellow" else "FAIL" + print(f" Salud: {color} {status.upper()}") + + nodes_info = requests.get(f"{ELASTIC_HOST}/_cat/nodes?h=name,ip,node.role,ram.percent,cpu&format=json", auth=('elastic', ELASTIC_PASSWORD), verify=False).json() + print(f" Nodos Activos ({len(nodes_info)}):") + for n in nodes_info: + print(f" - {n['name'].ljust(15)} | IP: {n['ip'].ljust(12)} | CPU: {n['cpu']}% | RAM: {n['ram.percent']}%") + + # Verificar ILM (R8.1) + ilm = requests.get(f"{ELASTIC_HOST}/_ilm/policy/siem-logs-policy", auth=('elastic', ELASTIC_PASSWORD), verify=False) + ilm_status = "APLICADA (30 días)" if ilm.status_code == 200 else "NO ENCONTRADA" + print(f" Política ILM: {ilm_status}") + except Exception as e: + print(f" Error conectando a Elasticsearch: {e}") + + # --- 2. SERVICIOS ELK (Logstash / Kibana) --- + print("\n[2/4] ESTADO DE SERVICIOS") + services = { + "Kibana": "https://kibana:5601/api/status", + "Logstash": "http://logstash:9600/_node/stats" + } + for name, url in services.items(): + try: + res = requests.get(url, timeout=3, verify=False) + status = "LIVE" if res.status_code == 200 else "STARTING" + print(f" └─ {name.ljust(10)}: {status}") + except: + print(f" └─ {name.ljust(10)}: DOWN / NOT REACHABLE") + + # --- 3. REGLAS DE SEGURIDAD (R8.4) --- + print("\n[3/4] REGLAS DE DETECCIÓN (SIEM)") + try: + # Consultar reglas vía API de Kibana + rules = requests.get("https://kibana:5601/api/detection_engine/rules/_find", + auth=('elastic', ELASTIC_PASSWORD), + headers={"kbn-xsrf": "true"}, timeout=5, verify=False).json() + total_rules = rules.get('total', 0) + mark = "OK" if total_rules >= 5 else "WARN" + print(f" {mark} Reglas Activas: {total_rules} (Mínimo requerido: 5)") + except: + print(" No se pudo conectar a la API de Reglas de Kibana.") + + # --- 4. INGESTA DE DATOS (R8.2) --- + print("\n[4/4] VOLUMEN DE DATOS") + sources = {"siem-auth": "Auth Logs", "siem-nginx": "Web Logs", "siem-syslog": "Syslogs"} + for idx, name in sources.items(): + try: + count = requests.get(f"{ELASTIC_HOST}/{idx}*/_count", auth=('elastic', ELASTIC_PASSWORD), verify=False).json().get('count', 0) + print(f" └─ {name.ljust(10)}: {count} docs") + except: + print(f" └─ {name.ljust(10)}: 0 docs") + + print("\n" + "="*70) + print("SUGERENCIA: Si los docs están en 0, ejecuta el generador de logs.") + print("="*70) + +if __name__ == "__main__": + run_diagnostic() diff --git a/setup/cleanup-indices.py b/setup/cleanup-indices.py new file mode 100644 index 0000000..290cbd7 --- /dev/null +++ b/setup/cleanup-indices.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +cleanup-indices.py +Elimina indices SIEM viejos que tengan mappings incorrectos (source.ip como text) +y verifica que los templates esten correctamente aplicados. + +Uso (manual - para arreglar el stack actual): + python3 setup/cleanup-indices.py + +Uso (docker - via ilm-setup): + python3 setup/cleanup-indices.py --docker +""" + +import os +import sys +import time + +try: + import requests + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + print("ERROR: pip install requests") + sys.exit(1) + +ES_URL = os.getenv("ELASTIC_HOSTS", "https://localhost:9200") +ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic") +ELASTIC_PASS = os.getenv("ELASTIC_PASSWORD", "SiemElastic2026!") +CACERT = os.getenv("CACERT", "./setup/certs/ca/ca.crt") +IS_DOCKER = "--docker" in sys.argv + +GREEN = "\033[92m" +RED = "\033[91m" +YELLOW = "\033[93m" +CYAN = "\033[96m" +BOLD = "\033[1m" +RESET = "\033[0m" + +TEMPLATES_CHECK = [ + ("siem-auth", "logs-auth-*", "source.ip"), + ("siem-nginx", "logs-nginx-*", "source.ip"), + ("siem-syslog", "logs-syslog-*", "source.ip"), + ("siem-k8s", "logs-k8s-*", None), + ("siem-errors", "logs-errors-*", None), +] + +INDICES_TO_CLEAN = [ + "logs-auth-*", "logs-nginx-*", "logs-syslog-*", + "logs-k8s-*", "logs-errors-*", +] + + +def es_session(): + s = requests.Session() + s.auth = (ELASTIC_USER, ELASTIC_PASS) + s.verify = CACERT if (CACERT and os.path.exists(CACERT)) else False + s.headers["Content-Type"] = "application/json" + return s + + +def wait_for_es(session, timeout=120): + deadline = time.time() + timeout + while time.time() < deadline: + try: + r = session.get(f"{ES_URL}/_cluster/health", timeout=5) + if r.status_code == 200: + status = r.json().get("status") + if status in ("green", "yellow"): + print(f" {GREEN}Cluster ES disponible (status={status}){RESET}") + return True + except Exception: + pass + time.sleep(3) + print(f" {RED}ES no disponible tras {timeout}s{RESET}") + return False + + +def disable_destructive_requires_name(session): + """ES 8.x bloquea DELETE con wildcards por defecto. Lo desactivamos.""" + try: + body = {"persistent": {"action.destructive_requires_name": False}} + r = session.put(f"{ES_URL}/_cluster/settings", json=body, timeout=10) + if r.status_code in (200, 201): + print(f" {GREEN}action.destructive_requires_name desactivado{RESET}") + else: + print(f" {YELLOW}No se pudo desactivar: {r.text[:100]}{RESET}") + except Exception as e: + print(f" {YELLOW}Error al desactivar: {e}{RESET}") + + +def delete_indices(session): + print(f"\n{CYAN}--- Eliminando indices logs-* con mapping incorrecto ---{RESET}") + disable_destructive_requires_name(session) + time.sleep(1) + all_ok = True + for pattern in INDICES_TO_CLEAN: + try: + r = session.delete(f"{ES_URL}/{pattern}") + if r.status_code in (200, 202): + print(f" {GREEN}Eliminado: {pattern}{RESET}") + elif r.status_code == 404: + print(f" {YELLOW}No existe: {pattern}{RESET}") + else: + print(f" {RED}Error {r.status_code} al eliminar {pattern}: {r.text[:100]}{RESET}") + all_ok = False + except Exception as e: + print(f" {RED}Excepcion al eliminar {pattern}: {e}{RESET}") + all_ok = False + + if all_ok: + print(f" {GREEN}Limpiando indices completado.{RESET}") + time.sleep(2) + return all_ok + + +def verify_templates(session): + """Verifica templates. No falla si faltan porque ilm-setup los crea despues.""" + print(f"\n{CYAN}--- Verificando templates de indice ---{RESET}") + + r = session.get(f"{ES_URL}/_index_template", timeout=10) + if r.status_code != 200: + print(f" {YELLOW}No se pudieron listar templates: {r.status_code}{RESET}") + return True + + existing = {t["name"] for t in r.json().get("index_templates", [])} + all_ok = True + + for tpl_name, idx_pattern, check_field in TEMPLATES_CHECK: + if tpl_name not in existing: + print(f" {YELLOW}Aun no existe: {tpl_name} (lo crea ilm-setup despues){RESET}") + else: + print(f" {GREEN}OK template: {tpl_name}{RESET}") + + print(f" {GREEN}Verificacion completada (ilm-setup creara los templates faltantes).{RESET}") + return True + + +def verify_mapping_on_new_index(session): + """Verifica que AL MENOS un indice SIEM exista con source.ip como keyword.""" + print(f"\n{CYAN}--- Verificando mapping en indices existentes ---{RESET}") + try: + r = session.get(f"{ES_URL}/_cat/indices/logs-*-*?h=index&format=json&s=index", timeout=10) + if r.status_code != 200 or not r.json(): + print(f" {YELLOW}Aun no hay indices SIEM (normal si se acaban de eliminar){RESET}") + return True + indices = [i["index"] for i in r.json()[:3]] + for idx in indices: + r2 = session.get(f"{ES_URL}/{idx}/_mapping", timeout=10) + if r2.status_code != 200: + continue + props = r2.json().get(idx, {}).get("mappings", {}).get("properties", {}) + source_ip = props.get("source", {}).get("properties", {}).get("ip", props.get("source.ip", {})) + if isinstance(source_ip, dict): + ip_type = source_ip.get("type", "?") + if ip_type == "keyword": + print(f" {GREEN}{idx}: source.ip={ip_type} OK{RESET}") + elif ip_type == "text": + print(f" {RED}{idx}: source.ip={ip_type} -> AUN MAL, reintentar cleanup{RESET}") + return False + except Exception as e: + print(f" {YELLOW}No se pudo verificar mapping: {e}{RESET}") + return True + + +def main(): + print(f"\n{BOLD}{'='*50}{RESET}") + print(f"{BOLD} Cleanup de indices SIEM - R8.3/R8.4{RESET}") + print(f" ES: {ES_URL}") + print(f"{BOLD}{'='*50}{RESET}") + + session = es_session() + + if not wait_for_es(session): + sys.exit(1) + + if not delete_indices(session): + print(f" {RED}Hubo errores eliminando indices. Continuando...{RESET}") + + if not verify_templates(session): + print(f" {RED}ERROR: Faltan templates. Revisar setup/{RESET}") + sys.exit(1) + + verify_mapping_on_new_index(session) + + print(f"\n{GREEN}Cleanup completado. Cuando Filebeat/Logstash procesen nuevos logs,") + print(f"los indices se crearan con mapping correcto (source.ip como keyword).{RESET}") + print(f"\n Recuerde generar logs: python setup/orchestrate-logs.py --mode random{RESET}") + print(f"O manual: python setup/generate-normal-logs.py --source all") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/setup/convert-rules-format.py b/setup/convert-rules-format.py new file mode 100644 index 0000000..9331c6a --- /dev/null +++ b/setup/convert-rules-format.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +""" +convert-rules-format.py +Convierte los archivos de reglas JSON a formato compatible con Kibana 8.19.14 +Renombra "id" a "rule_id" en cada regla. +""" + +import json +from pathlib import Path + +def convert_rules(): + rules_dir = Path(__file__).resolve().parent.parent / "rules" + + # Archivos de reglas individuales + rule_files = [ + "brute-force-ssh.json", + "login-after-hours.json", + "multiple-404.json", + "port-scan.json", + "sensitive-paths.json" + ] + + all_rules = [] + + for rule_file_name in rule_files: + rule_path = rules_dir / rule_file_name + if not rule_path.exists(): + print(f"WARN: {rule_file_name} no encontrado") + continue + + with rule_path.open("r") as f: + rule = json.load(f) + + # Renombrar "id" a "rule_id" + if "id" in rule: + rule["rule_id"] = rule.pop("id") + + all_rules.append(rule) + print(f"✓ Convertida: {rule_file_name}") + + # Generar NDJSON + output_path = rules_dir / "all-rules.ndjson" + with output_path.open("w") as f: + for rule in all_rules: + f.write(json.dumps(rule, ensure_ascii=False) + "\n") + + print(f"\n✓ Archivo generado: {output_path}") + print(f" Total de reglas: {len(all_rules)}") + +if __name__ == "__main__": + convert_rules() diff --git a/setup/create-dashboards-complete.py b/setup/create-dashboards-complete.py new file mode 100644 index 0000000..447dd80 --- /dev/null +++ b/setup/create-dashboards-complete.py @@ -0,0 +1,1159 @@ +#!/usr/bin/env python3 +""" +create-dashboards-complete.py +Genera dashboards ejecutivo y operacional con visualizaciones completas para Kibana 8.19.14 +""" + +import json +from pathlib import Path + +DASHBOARD_VERSION = "8.19.14" +LOGS_INDEX = "logs-*" + +RISK_QUERY = { + "bool": { + "filter": [ + {"terms": {"siem.severity.keyword": ["high", "medium"]}} + ] + } +} + +HIGH_RISK_QUERY = {"term": {"siem.severity.keyword": "high"}} +MEDIUM_RISK_QUERY = {"term": {"siem.severity.keyword": "medium"}} +AUTH_FAILURE_QUERY = {"term": {"event.action.keyword": "authentication_failure"}} +FIREWALL_BLOCK_QUERY = {"term": {"event.action.keyword": "firewall_block"}} +WEB_EVENTS_QUERY = {"term": {"event.dataset.keyword": "nginx.access"}} +AUTH_USER_QUERY = { + "bool": { + "filter": [ + {"term": {"event.dataset.keyword": "system.auth"}}, + {"exists": {"field": "user.name.keyword"}}, + ], + "must_not": [{"term": {"user.name.keyword": ""}}], + } +} + +THREAT_TAGS_REGEX = ( + "sqli_detected|path_traversal|security_scanner|sensitive_path_access|" + "auth_failure|firewall_block|after_hours_login|high_risk_sudo|" + "xss_detected|max_attempts_exceeded|http_404" +) +WEB_INDICATOR_TAGS_REGEX = ( + "sqli_detected|path_traversal|security_scanner|sensitive_path_access|" + "xss_detected|http_404" +) + + +def search_source(query=None, sort=None): + source = { + "index": LOGS_INDEX, + "query": query or {"match_all": {}}, + "filter": [], + } + if sort: + source["sort"] = sort + return {"searchSourceJSON": json.dumps(source, ensure_ascii=False)} + + +def build_metric_visualization(viz_id, title, sub_text, query=None, agg_type="count", agg_params=None): + return { + "type": "visualization", + "id": viz_id, + "attributes": { + "title": title, + "visState": json.dumps( + { + "title": title, + "type": "metric", + "params": { + "addTooltip": True, + "addLegend": False, + "type": "metric", + "metric": { + "percentageMode": False, + "useRanges": False, + "colorSchema": "Green to Red", + "metricColorMode": "None", + "colorsRange": [{"from": 0, "to": 10000}], + "labels": {"show": False}, + "invertColors": False, + "style": { + "bgFill": "#000", + "bgColor": False, + "labelColor": False, + "subText": sub_text, + "fontSize": 48, + }, + }, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": agg_type, + "schema": "metric", + "params": agg_params or {}, + } + ], + }, + ensure_ascii=False, + ), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(query), + }, + } + + +def build_pie_visualization(viz_id, title, field, query=None, size=8, include=None): + bucket_params = { + "field": field, + "size": size, + "order": "desc", + "orderBy": "1", + } + if include: + bucket_params["include"] = include + + return { + "type": "visualization", + "id": viz_id, + "attributes": { + "title": title, + "visState": json.dumps( + { + "title": title, + "type": "pie", + "params": { + "addLegend": True, + "addTooltip": True, + "isDonut": True, + "legendPosition": "right", + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {}, + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "segment", + "params": bucket_params, + }, + ], + }, + ensure_ascii=False, + ), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(query), + }, + } + + +def build_table_visualization(viz_id, title, buckets, query=None, per_page=10, sort=None): + aggs = [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {}, + } + ] + + for index, bucket in enumerate(buckets, start=2): + params = { + "field": bucket["field"], + "size": bucket.get("size", 10), + "order": bucket.get("order", "desc"), + "orderBy": bucket.get("orderBy", "1"), + } + if "include" in bucket: + params["include"] = bucket["include"] + aggs.append( + { + "id": str(index), + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": params, + } + ) + + return { + "type": "visualization", + "id": viz_id, + "attributes": { + "title": title, + "visState": json.dumps( + { + "title": title, + "type": "table", + "params": { + "perPage": per_page, + "showTotal": False, + }, + "aggs": aggs, + }, + ensure_ascii=False, + ), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(query, sort=sort), + }, + } + + +def build_line_visualization(viz_id, title, query=None, interval="m", custom_interval="1m"): + return { + "type": "visualization", + "id": viz_id, + "attributes": { + "title": title, + "visState": json.dumps( + { + "title": title, + "type": "line", + "params": { + "grid": {"categoryLines": False, "valueAxis": "ValueAxis-1"}, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "type": "category", + "position": "bottom", + "show": True, + "style": {}, + "scale": {"type": "linear"}, + "labels": {"show": True, "truncate": 100}, + "title": {}, + } + ], + "valueAxes": [ + { + "id": "ValueAxis-1", + "name": "LeftAxis-1", + "type": "value", + "position": "left", + "show": True, + "style": {}, + "scale": {"type": "linear", "mode": "normal"}, + "labels": {"show": True, "truncate": 100}, + "title": {"text": "Count"}, + } + ], + "seriesParams": [ + { + "show": True, + "type": "line", + "mode": "normal", + "data": {"label": "Count", "id": "1"}, + "interpolate": "linear", + "drawLinesBetweenPoints": True, + "showCircles": True, + "valueAxis": "ValueAxis-1", + } + ], + "addLegend": True, + "addTooltip": True, + "legendPosition": "bottom", + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {}, + }, + { + "id": "2", + "enabled": True, + "type": "date_histogram", + "schema": "segment", + "params": { + "field": "@timestamp", + "interval": interval, + "customInterval": custom_interval, + "min_doc_count": 1, + "extended_bounds": {}, + "order": "asc", + "orderBy": "_key", + }, + }, + ], + }, + ensure_ascii=False, + ), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(query), + }, + } + + +def build_dashboard( + dashboard_id, + title, + description, + panel_specs, + refresh_interval={"pause": True, "value": 0}, + time_from=None, + time_to=None, +): + panels = [] + references = [] + + for index, panel_spec in enumerate(panel_specs, start=1): + panel_ref_name = f"panel_{index}" + panels.append( + { + "version": DASHBOARD_VERSION, + "type": "visualization", + "gridData": panel_spec["gridData"], + "panelIndex": str(index), + "embeddableConfig": {}, + "panelRefName": panel_ref_name, + } + ) + references.append( + { + "type": "visualization", + "id": panel_spec["id"], + "name": panel_ref_name, + } + ) + + attributes = { + "title": title, + "description": description, + "optionsJSON": json.dumps( + { + "useMargins": True, + "syncColors": False, + "syncCursor": True, + "syncTooltips": False, + "hidePanelTitles": False, + } + ), + "panelsJSON": json.dumps(panels, ensure_ascii=False), + "refreshInterval": refresh_interval, + "timeRestore": bool(time_from or time_to), + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps( + { + "query": {"language": "kuery", "query": ""}, + "filter": [], + } + ) + }, + } + if time_from: + attributes["timeFrom"] = time_from + if time_to: + attributes["timeTo"] = time_to + + return { + "type": "dashboard", + "id": dashboard_id, + "attributes": attributes, + "references": references, + } + +def create_complete_dashboards(): + """Crea dashboards con visualizaciones para Kibana 8.19.14""" + + saved_objects = [] + + # Index pattern para logs + index_pattern = { + "type": "index-pattern", + "id": "logs-*", + "attributes": { + "title": "logs-*", + "timeFieldName": "@timestamp" + } + } + saved_objects.append(index_pattern) + saved_objects.extend( + [ + build_metric_visualization( + "viz-exec-total-events", + "Eventos Observados", + "últimas 24h", + ), + build_metric_visualization( + "viz-exec-high-risk", + "Riesgo Alto", + "eventos críticos", + query=HIGH_RISK_QUERY, + ), + build_metric_visualization( + "viz-exec-medium-risk", + "Riesgo Medio", + "eventos relevantes", + query=MEDIUM_RISK_QUERY, + ), + build_metric_visualization( + "viz-exec-risk-sources", + "IPs Sospechosas", + "orígenes únicos", + query=RISK_QUERY, + agg_type="cardinality", + agg_params={"field": "source.ip.keyword"}, + ), + ] + ) + + # ─── VISUALIZATIONS ────────────────────────────────────────────────────── + + # 1. Top Threats (pie chart) + viz_top_threats = { + "type": "visualization", + "id": "viz-top-threats", + "attributes": { + "title": "Top Señales de Riesgo del Día", + "visState": json.dumps({ + "title": "Top Señales de Riesgo del Día", + "type": "pie", + "params": { + "addLegend": True, + "addTooltip": True, + "isDonut": True, + "legendPosition": "right" + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "segment", + "params": { + "field": "tags.keyword", + "size": 8, + "include": THREAT_TAGS_REGEX, + "order": "desc", + "orderBy": "1" + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(RISK_QUERY) + } + } + saved_objects.append(viz_top_threats) + + # 2. Alert Trend (line chart) + viz_alert_trend = { + "type": "visualization", + "id": "viz-alert-trend", + "attributes": { + "title": "Tendencia de Riesgo por Hora", + "visState": json.dumps({ + "title": "Tendencia de Riesgo por Hora", + "type": "line", + "params": { + "grid": {"categoryLines": False, "valueAxis": "ValueAxis-1"}, + "categoryAxes": [{"id": "CategoryAxis-1", "type": "category", "position": "bottom", "show": True, "style": {}, "scale": {"type": "linear"}, "labels": {"show": True, "truncate": 100}, "title": {}}], + "valueAxes": [{"id": "ValueAxis-1", "name": "LeftAxis-1", "type": "value", "position": "left", "show": True, "style": {}, "scale": {"type": "linear", "mode": "normal"}, "labels": {"show": True, "truncate": 100}, "title": {"text": "Count"}}], + "seriesParams": [ + { + "show": True, + "type": "line", + "mode": "normal", + "data": {"label": "Count", "id": "1"}, + "interpolate": "linear", + "drawLinesBetweenPoints": True, + "showCircles": True, + "valueAxis": "ValueAxis-1", + } + ], + "addLegend": True, + "addTooltip": True, + "legendPosition": "bottom" + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "date_histogram", + "schema": "segment", + "params": { + "field": "@timestamp", + "interval": "h", + "customInterval": "1h", + "min_doc_count": 1, + "extended_bounds": {}, + "order": "asc", + "orderBy": "_key" + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(RISK_QUERY) + } + } + saved_objects.append(viz_alert_trend) + + # 3. Top IPs (Table) + viz_top_ips = { + "type": "visualization", + "id": "viz-top-ips", + "attributes": { + "title": "Top IPs con más Eventos", + "visState": json.dumps({ + "title": "Top IPs con más Eventos", + "type": "table", + "params": { + "perPage": 10, + "showTotal": False, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "source.ip.keyword", + "size": 10, + "order": "desc", + "orderBy": "1" + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": "logs-*", + "query": {"match_all": {}}, + "filter": [] + }) + } + } + } + saved_objects.append(viz_top_ips) + + # 4. System Health KPI (Gauge) + viz_health = { + "type": "visualization", + "id": "viz-system-health", + "attributes": { + "title": "Salud General: Hosts Monitoreados", + "visState": json.dumps({ + "title": "Salud General: Hosts Monitoreados", + "type": "gauge", + "params": { + "type": "gauge", + "addTooltip": True, + "addLegend": False, + "isDonut": True, + "gauge": { + "verticalSplit": False, + "extendRange": True, + "percentageMode": False, + "gaugeType": "Arc", + "gaugeStyle": "Full", + "backStyle": "Full", + "orientation": "vertical", + "colorSchema": "Green to Red", + "gaugeColorMode": "Labels", + "colorsRange": [ + {"from": 0, "to": 1}, + {"from": 1, "to": 3}, + {"from": 3, "to": 10} + ], + "invertColors": True, + "labels": {"show": True, "color": "black"}, + "scale": {"show": True, "labels": False, "color": "#333"}, + "type": "meter", + "style": { + "bgWidth": 0.9, + "width": 0.9, + "mask": False, + "bgMask": False, + "maskBars": 50, + "bgFill": "#eee", + "bgColor": False, + "subText": "Hosts", + "fontSize": 60 + } + } + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "cardinality", + "schema": "metric", + "params": {"field": "host.name.keyword"} + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": "logs-*", + "query": {"match_all": {}}, + "filter": [] + }) + } + } + } + saved_objects.append(viz_health) + + # Map + viz_geoip_map = { + "type": "visualization", + "id": "viz-geoip-map", + "attributes": { + "title": "Mapa Geográfico de IPs Sospechosas", + "visState": json.dumps({ + "title": "Mapa Geográfico de IPs Sospechosas", + "type": "tile_map", + "params": { + "addTooltip": True, + "mapType": "Scaled Circle Markers", + "isDesaturated": True, + "colorSchema": "Yellow to Red", + "heatMaxZoom": 16, + "heatMinOpacity": 0.1, + "heatRadius": 25, + "heatBlur": 15, + "heatNormalizeData": True + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "geohash_grid", + "schema": "segment", + "params": { + "field": "source.geo.location", + "autoPrecision": True + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(RISK_QUERY) + } + } + saved_objects.append(viz_geoip_map) + + # 5. Real-time Events Table (Detailed) + viz_events_table = { + "type": "visualization", + "id": "viz-events-table", + "attributes": { + "title": "Logs en Tiempo Real (Detalle)", + "visState": json.dumps({ + "title": "Logs en Tiempo Real (Detalle)", + "type": "table", + "params": { + "perPage": 20, + "showTotal": False, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "@timestamp", + "size": 50, + "order": "desc", + "orderBy": "_key" + } + }, + { + "id": "3", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "event.dataset.keyword", + "size": 5 + } + }, + { + "id": "4", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "message.keyword", + "size": 5 + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": "logs-*", + "query": {"match_all": {}}, + "filter": [], + "sort": [{"@timestamp": {"order": "desc"}}] + }) + } + } + } + saved_objects.append(viz_events_table) + + # 6. Active Alerts Table (Detailed) + viz_alerts_table = { + "type": "visualization", + "id": "viz-alerts-table", + "attributes": { + "title": "Alertas Activas Detalladas", + "visState": json.dumps({ + "title": "Alertas Activas Detalladas", + "type": "table", + "params": { + "perPage": 10, + "showTotal": False, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "siem.severity.keyword", + "size": 5 + } + }, + { + "id": "3", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "event.action.keyword", + "size": 10 + } + }, + { + "id": "4", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "event.dataset.keyword", + "size": 5 + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": "logs-*", + "query": { + "bool": { + "should": [ + {"term": {"siem.severity.keyword": "high"}}, + {"term": {"siem.severity.keyword": "medium"}}, + {"term": {"siem.severity.keyword": "low"}} + ], + "minimum_should_match": 1 + } + }, + "filter": [], + "sort": [{"@timestamp": {"order": "desc"}}] + }) + } + } + } + saved_objects.append(viz_alerts_table) + + # 7. Top Users (Table) + viz_top_users = { + "type": "visualization", + "id": "viz-top-users", + "attributes": { + "title": "Top Usuarios con más Eventos", + "visState": json.dumps({ + "title": "Top Usuarios con más Eventos", + "type": "table", + "params": { + "perPage": 10, + "showTotal": False, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "user.name.keyword", + "size": 10, + "order": "desc", + "orderBy": "1" + } + } + ] + }), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": "logs-*", + "query": {"match_all": {}}, + "filter": [] + }) + } + } + } + saved_objects.append(viz_top_users) + + # 8. Executive severity distribution + viz_exec_severity = { + "type": "visualization", + "id": "viz-exec-severity-distribution", + "attributes": { + "title": "Distribución por Severidad", + "visState": json.dumps({ + "title": "Distribución por Severidad", + "type": "pie", + "params": { + "addLegend": True, + "addTooltip": True, + "isDonut": True, + "legendPosition": "right" + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "segment", + "params": { + "field": "siem.severity.keyword", + "size": 4, + "order": "desc", + "orderBy": "1" + } + } + ] + }, ensure_ascii=False), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source() + } + } + saved_objects.append(viz_exec_severity) + + # 9. Executive source coverage + viz_exec_source_coverage = { + "type": "visualization", + "id": "viz-exec-source-coverage", + "attributes": { + "title": "Cobertura por Fuente de Log", + "visState": json.dumps({ + "title": "Cobertura por Fuente de Log", + "type": "pie", + "params": { + "addLegend": True, + "addTooltip": True, + "isDonut": True, + "legendPosition": "right" + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "segment", + "params": { + "field": "event.dataset.keyword", + "size": 8, + "order": "desc", + "orderBy": "1" + } + } + ] + }, ensure_ascii=False), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source() + } + } + saved_objects.append(viz_exec_source_coverage) + + # 10. Executive suspicious IPs table + viz_exec_risk_ips = { + "type": "visualization", + "id": "viz-exec-risk-ips", + "attributes": { + "title": "Top IPs Sospechosas", + "visState": json.dumps({ + "title": "Top IPs Sospechosas", + "type": "table", + "params": { + "perPage": 10, + "showTotal": False, + }, + "aggs": [ + { + "id": "1", + "enabled": True, + "type": "count", + "schema": "metric", + "params": {} + }, + { + "id": "2", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "source.ip.keyword", + "size": 10, + "order": "desc", + "orderBy": "1" + } + }, + { + "id": "3", + "enabled": True, + "type": "terms", + "schema": "bucket", + "params": { + "field": "event.action.keyword", + "size": 5, + "order": "desc", + "orderBy": "1" + } + } + ] + }, ensure_ascii=False), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": search_source(RISK_QUERY) + } + } + saved_objects.append(viz_exec_risk_ips) + + # ─── OPERATIONAL TRIAGE VISUALIZATIONS ────────────────────────────────── + saved_objects.extend( + [ + build_metric_visualization( + "viz-op-events-recent", + "Eventos Recientes", + "últimas 2h", + ), + build_metric_visualization( + "viz-op-risk-events", + "Eventos de Riesgo", + "high + medium", + query=RISK_QUERY, + ), + build_metric_visualization( + "viz-op-ssh-failures", + "Fallos SSH", + "authentication_failure", + query=AUTH_FAILURE_QUERY, + ), + build_metric_visualization( + "viz-op-firewall-blocks", + "Bloqueos Firewall", + "firewall_block", + query=FIREWALL_BLOCK_QUERY, + ), + build_pie_visualization( + "viz-op-actions", + "Acciones Detectadas", + "event.action.keyword", + query=RISK_QUERY, + size=8, + ), + build_pie_visualization( + "viz-op-web-indicators", + "Indicadores Web", + "tags.keyword", + query=WEB_EVENTS_QUERY, + size=8, + include=WEB_INDICATOR_TAGS_REGEX, + ), + build_line_visualization( + "viz-op-risk-trend", + "Tendencia de Riesgo por Minuto", + query=RISK_QUERY, + interval="m", + custom_interval="1m", + ), + build_table_visualization( + "viz-op-top-auth-users", + "Top Usuarios Auth", + [ + {"field": "user.name.keyword", "size": 10}, + {"field": "event.action.keyword", "size": 5}, + {"field": "source.ip.keyword", "size": 5}, + ], + query=AUTH_USER_QUERY, + per_page=10, + ), + build_table_visualization( + "viz-op-blocked-ports", + "Puertos Destino Bloqueados", + [ + {"field": "destination.port", "size": 12}, + {"field": "source.ip.keyword", "size": 5}, + ], + query=FIREWALL_BLOCK_QUERY, + per_page=12, + ), + build_table_visualization( + "viz-op-investigation-events", + "Eventos de Investigación", + [ + {"field": "@timestamp", "size": 50, "orderBy": "_key"}, + {"field": "siem.severity.keyword", "size": 4}, + {"field": "event.dataset.keyword", "size": 6}, + {"field": "event.action.keyword", "size": 8}, + {"field": "source.ip.keyword", "size": 10}, + {"field": "message.keyword", "size": 5}, + ], + query=RISK_QUERY, + per_page=20, + sort=[{"@timestamp": {"order": "desc"}}], + ), + ] + ) + + # ─── DASHBOARDS ────────────────────────────────────────────────────────── + + executive_panels = [ + {"id": "viz-exec-total-events", "gridData": {"x": 0, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-exec-high-risk", "gridData": {"x": 12, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-exec-medium-risk", "gridData": {"x": 24, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-exec-risk-sources", "gridData": {"x": 36, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-system-health", "gridData": {"x": 0, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-exec-severity-distribution", "gridData": {"x": 16, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-exec-source-coverage", "gridData": {"x": 32, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-top-threats", "gridData": {"x": 0, "y": 24, "w": 24, "h": 15}}, + {"id": "viz-alert-trend", "gridData": {"x": 24, "y": 24, "w": 24, "h": 15}}, + {"id": "viz-geoip-map", "gridData": {"x": 0, "y": 39, "w": 24, "h": 18}}, + {"id": "viz-exec-risk-ips", "gridData": {"x": 24, "y": 39, "w": 24, "h": 18}}, + ] + saved_objects.append( + build_dashboard( + "executive-security-overview", + "Executive - Resumen de Seguridad", + "Panel ejecutivo con KPIs, amenazas, cobertura y tendencia de riesgo para audiencia no técnica", + executive_panels, + refresh_interval={"pause": False, "value": 10000}, + time_from="now-24h", + time_to="now", + ) + ) + + operational_panels = [ + {"id": "viz-op-events-recent", "gridData": {"x": 0, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-op-risk-events", "gridData": {"x": 12, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-op-ssh-failures", "gridData": {"x": 24, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-op-firewall-blocks", "gridData": {"x": 36, "y": 0, "w": 12, "h": 10}}, + {"id": "viz-exec-severity-distribution", "gridData": {"x": 0, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-op-actions", "gridData": {"x": 16, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-op-web-indicators", "gridData": {"x": 32, "y": 10, "w": 16, "h": 14}}, + {"id": "viz-op-risk-trend", "gridData": {"x": 0, "y": 24, "w": 24, "h": 15}}, + {"id": "viz-op-blocked-ports", "gridData": {"x": 24, "y": 24, "w": 24, "h": 15}}, + {"id": "viz-exec-risk-ips", "gridData": {"x": 0, "y": 39, "w": 24, "h": 18}}, + {"id": "viz-op-top-auth-users", "gridData": {"x": 24, "y": 39, "w": 24, "h": 18}}, + {"id": "viz-op-investigation-events", "gridData": {"x": 0, "y": 57, "w": 48, "h": 22}}, + ] + saved_objects.append( + build_dashboard( + "operational-triage-console", + "Operational - Consola de Triaje", + "Panel operacional para investigación y respuesta a incidentes", + operational_panels, + refresh_interval={"pause": False, "value": 10000}, + time_from="now-2h", + time_to="now", + ) + ) + + # Write NDJSON + output_path = Path(__file__).resolve().parent.parent / "setup" / "dashboards" / "executive-operational-dashboards.ndjson" + output_path.parent.mkdir(parents=True, exist_ok=True) + + with output_path.open("w", encoding="utf-8") as f: + for item in saved_objects: + f.write(json.dumps(item, ensure_ascii=False) + "\n") + + print(f"✓ Dashboards completos creados para Kibana 8.19.14") + print(f" Archivo: {output_path}") + print(f" Visualizaciones: {sum(1 for d in saved_objects if d['type'] == 'visualization')}") + print(f" Dashboards: {sum(1 for d in saved_objects if d['type'] == 'dashboard')}") + +if __name__ == "__main__": + create_complete_dashboards() diff --git a/setup/create-data-views.py b/setup/create-data-views.py new file mode 100644 index 0000000..2f13e8e --- /dev/null +++ b/setup/create-data-views.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Crea los data views necesarios en Kibana si no existen.""" + +import os +import sys +import time +from pathlib import Path + +try: + import requests + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + print("ERROR: pip install requests") + sys.exit(1) + +KIBANA_URL = os.getenv("KIBANA_HOST", "https://localhost:5601") +ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic") +ELASTIC_PASS = os.getenv("ELASTIC_PASSWORD", "SiemElastic2026!") +CACERT = os.getenv("CACERT", "./setup/certs/ca/ca.crt") +WAIT_KIBANA_TIMEOUT = int(os.getenv("WAIT_KIBANA_TIMEOUT", "600")) +DATA_VIEW_RETRIES = int(os.getenv("DATA_VIEW_RETRIES", "5")) +DATA_VIEW_RETRY_WAIT = int(os.getenv("DATA_VIEW_RETRY_WAIT", "10")) + +DATA_VIEWS = [ + {"id": "logs-*", "title": "logs-*", "description": "Todos los logs"}, + {"id": "logs-auth-*", "title": "logs-auth-*", "description": "Logs de autenticacion"}, + {"id": "logs-nginx-*", "title": "logs-nginx-*", "description": "Logs de nginx/apache"}, + {"id": "logs-syslog-*","title": "logs-syslog-*","description": "Logs de sistema"}, + {"id": "logs-k8s-*", "title": "logs-k8s-*", "description": "Logs de Kubernetes"}, +] + +G = "\033[92m" +R = "\033[91m" +Y = "\033[93m" +B = "\033[94m" +W = "\033[1m" +X = "\033[0m" + + +def kibana_session(): + client = requests.Session() + client.auth = (ELASTIC_USER, ELASTIC_PASS) + client.headers = {"kbn-xsrf": "true"} + client.verify = CACERT if Path(CACERT).exists() else False + return client + + +def wait_for_kibana(client, timeout=WAIT_KIBANA_TIMEOUT): + print(f"\n{W}Esperando a que Kibana este disponible...{X}") + deadline = time.time() + timeout + while time.time() < deadline: + try: + response = client.get(f"{KIBANA_URL}/api/status", timeout=10) + if response.status_code == 200: + data = response.json() + overall = data.get("status", {}).get("overall", {}) + level = overall.get("level") + if level in ("available", "green"): + print(f" {G}✓{X} Kibana disponible") + return True + except Exception: + pass + time.sleep(5) + print(f" {R}✗{X} Kibana no respondio en {timeout}s") + return False + + +def ensure_data_view(client, data_view_id, title): + get_url = f"{KIBANA_URL}/api/data_views/data_view/{data_view_id}" + create_url = f"{KIBANA_URL}/api/data_views/data_view" + + for attempt in range(1, DATA_VIEW_RETRIES + 1): + try: + response = client.get(get_url, timeout=10) + if response.status_code == 200: + print(f" {G}✓{X} Data view existe: {data_view_id}") + return True + if response.status_code not in (404, 400): + print(f" {R}✗{X} Error consultando data view: {response.status_code}") + print(response.text[:500]) + return False + + payload = { + "data_view": { + "id": data_view_id, + "title": title, + "timeFieldName": "@timestamp", + } + } + created = client.post(create_url, json=payload, timeout=20) + if created.status_code in (200, 201): + print(f" {G}✓{X} Data view creado: {data_view_id}") + return True + + print(f" {Y}⚠{X} Error creando data view (intento {attempt}/{DATA_VIEW_RETRIES})") + print(created.text[:300]) + except Exception: + print(f" {Y}⚠{X} Error temporal creando data view (intento {attempt}/{DATA_VIEW_RETRIES})") + + time.sleep(DATA_VIEW_RETRY_WAIT) + + return False + + +def main(): + print(f"\n{W}=== Creando Data Views en Kibana ==={X}") + client = kibana_session() + + if not wait_for_kibana(client): + sys.exit(1) + + all_ok = True + for dv in DATA_VIEWS: + print(f"\n {B}-- {dv['description']}{X}") + if not ensure_data_view(client, dv["id"], dv["title"]): + print(f" {R}✗{X} Fallo al crear data view: {dv['id']}") + all_ok = False + + if all_ok: + print(f"\n {G}✓{X} Todos los data views listos") + else: + print(f"\n {Y}⚠{X} Algunos data views no se crearon correctamente") + + +if __name__ == "__main__": + main() diff --git a/setup/create_dashboards.py b/setup/create_dashboards.py new file mode 100644 index 0000000..9eb9884 --- /dev/null +++ b/setup/create_dashboards.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import os +import ssl +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +DEFAULT_BASE_URL = "https://localhost:5601" +DEFAULT_USER = "elastic" +DEFAULT_PASSWORD = "SiemElastic2026!" +DEFAULT_SOURCE_QUERY = "CyberArk" +DEFAULT_OUTPUT = Path("setup/dashboards/executive-operational-dashboards.ndjson") + + +def _build_request(base_url, path, user, password, method="GET", body=None): + url = base_url.rstrip("/") + path + headers = {"kbn-xsrf": "true"} + if body is not None: + headers["Content-Type"] = "application/json" + token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + data = None if body is None else json.dumps(body).encode("utf-8") + return urllib.request.Request(url, data=data, headers=headers, method=method) + + +def request_json(base_url, path, user, password, method="GET", body=None, insecure=False): + context = ssl._create_unverified_context() if insecure else None + request = _build_request(base_url, path, user, password, method=method, body=body) + with urllib.request.urlopen(request, context=context) as response: + payload = response.read().decode("utf-8") + content_type = response.headers.get("Content-Type", "") + if "application/json" in content_type or payload.startswith("{") or payload.startswith("["): + return json.loads(payload) + return payload + + +def request_text(base_url, path, user, password, method="GET", body=None, insecure=False): + context = ssl._create_unverified_context() if insecure else None + request = _build_request(base_url, path, user, password, method=method, body=body) + with urllib.request.urlopen(request, context=context) as response: + return response.read().decode("utf-8") + + +def find_source_dashboard(base_url, user, password, search_text, insecure=False): + result = request_json( + base_url, + f"/api/saved_objects/_find?type=dashboard&search={urllib.parse.quote(search_text)}&search_fields=title&per_page=1", + user, + password, + insecure=insecure, + ) + objects = result.get("saved_objects", []) + if not objects: + raise RuntimeError(f"No dashboard found matching {search_text!r}") + return objects[0] + + +def clone_dashboard(source, new_id, new_title, new_description): + clone = { + "type": "dashboard", + "id": new_id, + "attributes": json.loads(json.dumps(source["attributes"])), + "references": json.loads(json.dumps(source.get("references", []))), + } + clone["attributes"]["title"] = new_title + clone["attributes"]["description"] = new_description + clone["attributes"]["timeRestore"] = False + return clone + + +def export_dashboards(base_url, user, password, dashboard_ids, output_path, insecure=False): + body = {"objects": [{"type": "dashboard", "id": dashboard_id} for dashboard_id in dashboard_ids], "includeReferencesDeep": True} + payload = request_text(base_url, "/api/saved_objects/_export", user, password, method="POST", body=body, insecure=insecure) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload, encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description="Clone a working Kibana dashboard into executive and operational views.") + parser.add_argument("--kibana", default=os.getenv("KIBANA_HOST", DEFAULT_BASE_URL), help="Kibana base URL") + parser.add_argument("--user", default=os.getenv("KIBANA_USER", DEFAULT_USER), help="Kibana username") + parser.add_argument("--password", default=os.getenv("KIBANA_PASSWORD", DEFAULT_PASSWORD), help="Kibana password") + parser.add_argument("--search", default=DEFAULT_SOURCE_QUERY, help="Dashboard title search text") + parser.add_argument("--output", default=str(DEFAULT_OUTPUT), help="NDJSON export path") + parser.add_argument("--insecure", action="store_true", help="Skip TLS certificate validation") + args = parser.parse_args() + + source = find_source_dashboard(args.kibana, args.user, args.password, args.search, insecure=args.insecure) + executive = clone_dashboard( + source, + "executive-security-overview", + "Executive Security Overview", + "Resumen ejecutivo de actividad, autenticaciones y señales de riesgo.", + ) + operational = clone_dashboard( + source, + "operational-triage-console", + "Operational Triage Console", + "Vista operacional para investigación, triage y respuesta.", + ) + + request_json(args.kibana, "/api/saved_objects/_bulk_create?overwrite=true", args.user, args.password, method="POST", body=[executive, operational], insecure=args.insecure) + export_dashboards(args.kibana, args.user, args.password, [executive["id"], operational["id"]], Path(args.output), insecure=args.insecure) + + print(f"Created dashboards: {executive['id']}, {operational['id']}") + print(f"Exported NDJSON to: {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup/dashboards/executive-operational-dashboards.ndjson b/setup/dashboards/executive-operational-dashboards.ndjson new file mode 100644 index 0000000..0d5e10a --- /dev/null +++ b/setup/dashboards/executive-operational-dashboards.ndjson @@ -0,0 +1,28 @@ +{"type": "index-pattern", "id": "logs-*", "attributes": {"title": "logs-*", "timeFieldName": "@timestamp"}} +{"type": "visualization", "id": "viz-exec-total-events", "attributes": {"title": "Eventos Observados", "visState": "{\"title\": \"Eventos Observados\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"últimas 24h\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-high-risk", "attributes": {"title": "Riesgo Alto", "visState": "{\"title\": \"Riesgo Alto\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"eventos críticos\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"siem.severity.keyword\": \"high\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-medium-risk", "attributes": {"title": "Riesgo Medio", "visState": "{\"title\": \"Riesgo Medio\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"eventos relevantes\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"siem.severity.keyword\": \"medium\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-risk-sources", "attributes": {"title": "IPs Sospechosas", "visState": "{\"title\": \"IPs Sospechosas\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"orígenes únicos\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"cardinality\", \"schema\": \"metric\", \"params\": {\"field\": \"source.ip.keyword\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-top-threats", "attributes": {"title": "Top Señales de Riesgo del Día", "visState": "{\"title\": \"Top Se\\u00f1ales de Riesgo del D\\u00eda\", \"type\": \"pie\", \"params\": {\"addLegend\": true, \"addTooltip\": true, \"isDonut\": true, \"legendPosition\": \"right\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"tags.keyword\", \"size\": 8, \"include\": \"sqli_detected|path_traversal|security_scanner|sensitive_path_access|auth_failure|firewall_block|after_hours_login|high_risk_sudo|xss_detected|max_attempts_exceeded|http_404\", \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-alert-trend", "attributes": {"title": "Tendencia de Riesgo por Hora", "visState": "{\"title\": \"Tendencia de Riesgo por Hora\", \"type\": \"line\", \"params\": {\"grid\": {\"categoryLines\": false, \"valueAxis\": \"ValueAxis-1\"}, \"categoryAxes\": [{\"id\": \"CategoryAxis-1\", \"type\": \"category\", \"position\": \"bottom\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\"}, \"labels\": {\"show\": true, \"truncate\": 100}, \"title\": {}}], \"valueAxes\": [{\"id\": \"ValueAxis-1\", \"name\": \"LeftAxis-1\", \"type\": \"value\", \"position\": \"left\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\", \"mode\": \"normal\"}, \"labels\": {\"show\": true, \"truncate\": 100}, \"title\": {\"text\": \"Count\"}}], \"seriesParams\": [{\"show\": true, \"type\": \"line\", \"mode\": \"normal\", \"data\": {\"label\": \"Count\", \"id\": \"1\"}, \"interpolate\": \"linear\", \"drawLinesBetweenPoints\": true, \"showCircles\": true, \"valueAxis\": \"ValueAxis-1\"}], \"addLegend\": true, \"addTooltip\": true, \"legendPosition\": \"bottom\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"date_histogram\", \"schema\": \"segment\", \"params\": {\"field\": \"@timestamp\", \"interval\": \"h\", \"customInterval\": \"1h\", \"min_doc_count\": 1, \"extended_bounds\": {}, \"order\": \"asc\", \"orderBy\": \"_key\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-top-ips", "attributes": {"title": "Top IPs con más Eventos", "visState": "{\"title\": \"Top IPs con m\\u00e1s Eventos\", \"type\": \"table\", \"params\": {\"perPage\": 10, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"source.ip.keyword\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-system-health", "attributes": {"title": "Salud General: Hosts Monitoreados", "visState": "{\"title\": \"Salud General: Hosts Monitoreados\", \"type\": \"gauge\", \"params\": {\"type\": \"gauge\", \"addTooltip\": true, \"addLegend\": false, \"isDonut\": true, \"gauge\": {\"verticalSplit\": false, \"extendRange\": true, \"percentageMode\": false, \"gaugeType\": \"Arc\", \"gaugeStyle\": \"Full\", \"backStyle\": \"Full\", \"orientation\": \"vertical\", \"colorSchema\": \"Green to Red\", \"gaugeColorMode\": \"Labels\", \"colorsRange\": [{\"from\": 0, \"to\": 1}, {\"from\": 1, \"to\": 3}, {\"from\": 3, \"to\": 10}], \"invertColors\": true, \"labels\": {\"show\": true, \"color\": \"black\"}, \"scale\": {\"show\": true, \"labels\": false, \"color\": \"#333\"}, \"type\": \"meter\", \"style\": {\"bgWidth\": 0.9, \"width\": 0.9, \"mask\": false, \"bgMask\": false, \"maskBars\": 50, \"bgFill\": \"#eee\", \"bgColor\": false, \"subText\": \"Hosts\", \"fontSize\": 60}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"cardinality\", \"schema\": \"metric\", \"params\": {\"field\": \"host.name.keyword\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-geoip-map", "attributes": {"title": "Mapa Geográfico de IPs Sospechosas", "visState": "{\"title\": \"Mapa Geogr\\u00e1fico de IPs Sospechosas\", \"type\": \"tile_map\", \"params\": {\"addTooltip\": true, \"mapType\": \"Scaled Circle Markers\", \"isDesaturated\": true, \"colorSchema\": \"Yellow to Red\", \"heatMaxZoom\": 16, \"heatMinOpacity\": 0.1, \"heatRadius\": 25, \"heatBlur\": 15, \"heatNormalizeData\": true}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"geohash_grid\", \"schema\": \"segment\", \"params\": {\"field\": \"source.geo.location\", \"autoPrecision\": true}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-events-table", "attributes": {"title": "Logs en Tiempo Real (Detalle)", "visState": "{\"title\": \"Logs en Tiempo Real (Detalle)\", \"type\": \"table\", \"params\": {\"perPage\": 20, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"@timestamp\", \"size\": 50, \"order\": \"desc\", \"orderBy\": \"_key\"}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.dataset.keyword\", \"size\": 5}}, {\"id\": \"4\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"message.keyword\", \"size\": 5}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": [], \"sort\": [{\"@timestamp\": {\"order\": \"desc\"}}]}"}}} +{"type": "visualization", "id": "viz-alerts-table", "attributes": {"title": "Alertas Activas Detalladas", "visState": "{\"title\": \"Alertas Activas Detalladas\", \"type\": \"table\", \"params\": {\"perPage\": 10, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"siem.severity.keyword\", \"size\": 5}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.action.keyword\", \"size\": 10}}, {\"id\": \"4\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.dataset.keyword\", \"size\": 5}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"should\": [{\"term\": {\"siem.severity.keyword\": \"high\"}}, {\"term\": {\"siem.severity.keyword\": \"medium\"}}, {\"term\": {\"siem.severity.keyword\": \"low\"}}], \"minimum_should_match\": 1}}, \"filter\": [], \"sort\": [{\"@timestamp\": {\"order\": \"desc\"}}]}"}}} +{"type": "visualization", "id": "viz-top-users", "attributes": {"title": "Top Usuarios con más Eventos", "visState": "{\"title\": \"Top Usuarios con m\\u00e1s Eventos\", \"type\": \"table\", \"params\": {\"perPage\": 10, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"user.name.keyword\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-severity-distribution", "attributes": {"title": "Distribución por Severidad", "visState": "{\"title\": \"Distribución por Severidad\", \"type\": \"pie\", \"params\": {\"addLegend\": true, \"addTooltip\": true, \"isDonut\": true, \"legendPosition\": \"right\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"siem.severity.keyword\", \"size\": 4, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-source-coverage", "attributes": {"title": "Cobertura por Fuente de Log", "visState": "{\"title\": \"Cobertura por Fuente de Log\", \"type\": \"pie\", \"params\": {\"addLegend\": true, \"addTooltip\": true, \"isDonut\": true, \"legendPosition\": \"right\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"event.dataset.keyword\", \"size\": 8, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-exec-risk-ips", "attributes": {"title": "Top IPs Sospechosas", "visState": "{\"title\": \"Top IPs Sospechosas\", \"type\": \"table\", \"params\": {\"perPage\": 10, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"source.ip.keyword\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.action.keyword\", \"size\": 5, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-events-recent", "attributes": {"title": "Eventos Recientes", "visState": "{\"title\": \"Eventos Recientes\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"últimas 2h\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"match_all\": {}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-risk-events", "attributes": {"title": "Eventos de Riesgo", "visState": "{\"title\": \"Eventos de Riesgo\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"high + medium\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-ssh-failures", "attributes": {"title": "Fallos SSH", "visState": "{\"title\": \"Fallos SSH\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"authentication_failure\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"event.action.keyword\": \"authentication_failure\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-firewall-blocks", "attributes": {"title": "Bloqueos Firewall", "visState": "{\"title\": \"Bloqueos Firewall\", \"type\": \"metric\", \"params\": {\"addTooltip\": true, \"addLegend\": false, \"type\": \"metric\", \"metric\": {\"percentageMode\": false, \"useRanges\": false, \"colorSchema\": \"Green to Red\", \"metricColorMode\": \"None\", \"colorsRange\": [{\"from\": 0, \"to\": 10000}], \"labels\": {\"show\": false}, \"invertColors\": false, \"style\": {\"bgFill\": \"#000\", \"bgColor\": false, \"labelColor\": false, \"subText\": \"firewall_block\", \"fontSize\": 48}}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"event.action.keyword\": \"firewall_block\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-actions", "attributes": {"title": "Acciones Detectadas", "visState": "{\"title\": \"Acciones Detectadas\", \"type\": \"pie\", \"params\": {\"addLegend\": true, \"addTooltip\": true, \"isDonut\": true, \"legendPosition\": \"right\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"event.action.keyword\", \"size\": 8, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-web-indicators", "attributes": {"title": "Indicadores Web", "visState": "{\"title\": \"Indicadores Web\", \"type\": \"pie\", \"params\": {\"addLegend\": true, \"addTooltip\": true, \"isDonut\": true, \"legendPosition\": \"right\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"tags.keyword\", \"size\": 8, \"order\": \"desc\", \"orderBy\": \"1\", \"include\": \"sqli_detected|path_traversal|security_scanner|sensitive_path_access|xss_detected|http_404\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"event.dataset.keyword\": \"nginx.access\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-risk-trend", "attributes": {"title": "Tendencia de Riesgo por Minuto", "visState": "{\"title\": \"Tendencia de Riesgo por Minuto\", \"type\": \"line\", \"params\": {\"grid\": {\"categoryLines\": false, \"valueAxis\": \"ValueAxis-1\"}, \"categoryAxes\": [{\"id\": \"CategoryAxis-1\", \"type\": \"category\", \"position\": \"bottom\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\"}, \"labels\": {\"show\": true, \"truncate\": 100}, \"title\": {}}], \"valueAxes\": [{\"id\": \"ValueAxis-1\", \"name\": \"LeftAxis-1\", \"type\": \"value\", \"position\": \"left\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\", \"mode\": \"normal\"}, \"labels\": {\"show\": true, \"truncate\": 100}, \"title\": {\"text\": \"Count\"}}], \"seriesParams\": [{\"show\": true, \"type\": \"line\", \"mode\": \"normal\", \"data\": {\"label\": \"Count\", \"id\": \"1\"}, \"interpolate\": \"linear\", \"drawLinesBetweenPoints\": true, \"showCircles\": true, \"valueAxis\": \"ValueAxis-1\"}], \"addLegend\": true, \"addTooltip\": true, \"legendPosition\": \"bottom\"}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"date_histogram\", \"schema\": \"segment\", \"params\": {\"field\": \"@timestamp\", \"interval\": \"m\", \"customInterval\": \"1m\", \"min_doc_count\": 1, \"extended_bounds\": {}, \"order\": \"asc\", \"orderBy\": \"_key\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-top-auth-users", "attributes": {"title": "Top Usuarios Auth", "visState": "{\"title\": \"Top Usuarios Auth\", \"type\": \"table\", \"params\": {\"perPage\": 10, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"user.name.keyword\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.action.keyword\", \"size\": 5, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"4\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"source.ip.keyword\", \"size\": 5, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"term\": {\"event.dataset.keyword\": \"system.auth\"}}, {\"exists\": {\"field\": \"user.name.keyword\"}}], \"must_not\": [{\"term\": {\"user.name.keyword\": \"\"}}]}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-blocked-ports", "attributes": {"title": "Puertos Destino Bloqueados", "visState": "{\"title\": \"Puertos Destino Bloqueados\", \"type\": \"table\", \"params\": {\"perPage\": 12, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"destination.port\", \"size\": 12, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"source.ip.keyword\", \"size\": 5, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"term\": {\"event.action.keyword\": \"firewall_block\"}}, \"filter\": []}"}}} +{"type": "visualization", "id": "viz-op-investigation-events", "attributes": {"title": "Eventos de Investigación", "visState": "{\"title\": \"Eventos de Investigación\", \"type\": \"table\", \"params\": {\"perPage\": 20, \"showTotal\": false}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"@timestamp\", \"size\": 50, \"order\": \"desc\", \"orderBy\": \"_key\"}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"siem.severity.keyword\", \"size\": 4, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"4\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.dataset.keyword\", \"size\": 6, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"5\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"event.action.keyword\", \"size\": 8, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"6\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"source.ip.keyword\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\"}}, {\"id\": \"7\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"bucket\", \"params\": {\"field\": \"message.keyword\", \"size\": 5, \"order\": \"desc\", \"orderBy\": \"1\"}}]}", "uiStateJSON": "{}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\": \"logs-*\", \"query\": {\"bool\": {\"filter\": [{\"terms\": {\"siem.severity.keyword\": [\"high\", \"medium\"]}}]}}, \"filter\": [], \"sort\": [{\"@timestamp\": {\"order\": \"desc\"}}]}"}}} +{"type": "dashboard", "id": "executive-security-overview", "attributes": {"title": "Executive - Resumen de Seguridad", "description": "Panel ejecutivo con KPIs, amenazas, cobertura y tendencia de riesgo para audiencia no técnica", "optionsJSON": "{\"useMargins\": true, \"syncColors\": false, \"syncCursor\": true, \"syncTooltips\": false, \"hidePanelTitles\": false}", "panelsJSON": "[{\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"1\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_1\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 12, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"2\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_2\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"3\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_3\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 36, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"4\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_4\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"5\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_5\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 16, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"6\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_6\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 32, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"7\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_7\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 24, \"w\": 24, \"h\": 15}, \"panelIndex\": \"8\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_8\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 24, \"w\": 24, \"h\": 15}, \"panelIndex\": \"9\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_9\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 39, \"w\": 24, \"h\": 18}, \"panelIndex\": \"10\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_10\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 39, \"w\": 24, \"h\": 18}, \"panelIndex\": \"11\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_11\"}]", "refreshInterval": {"pause": false, "value": 10000}, "timeRestore": true, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"query\": {\"language\": \"kuery\", \"query\": \"\"}, \"filter\": []}"}, "timeFrom": "now-24h", "timeTo": "now"}, "references": [{"type": "visualization", "id": "viz-exec-total-events", "name": "panel_1"}, {"type": "visualization", "id": "viz-exec-high-risk", "name": "panel_2"}, {"type": "visualization", "id": "viz-exec-medium-risk", "name": "panel_3"}, {"type": "visualization", "id": "viz-exec-risk-sources", "name": "panel_4"}, {"type": "visualization", "id": "viz-system-health", "name": "panel_5"}, {"type": "visualization", "id": "viz-exec-severity-distribution", "name": "panel_6"}, {"type": "visualization", "id": "viz-exec-source-coverage", "name": "panel_7"}, {"type": "visualization", "id": "viz-top-threats", "name": "panel_8"}, {"type": "visualization", "id": "viz-alert-trend", "name": "panel_9"}, {"type": "visualization", "id": "viz-geoip-map", "name": "panel_10"}, {"type": "visualization", "id": "viz-exec-risk-ips", "name": "panel_11"}]} +{"type": "dashboard", "id": "operational-triage-console", "attributes": {"title": "Operational - Consola de Triaje", "description": "Panel operacional para investigación y respuesta a incidentes", "optionsJSON": "{\"useMargins\": true, \"syncColors\": false, \"syncCursor\": true, \"syncTooltips\": false, \"hidePanelTitles\": false}", "panelsJSON": "[{\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"1\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_1\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 12, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"2\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_2\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"3\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_3\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 36, \"y\": 0, \"w\": 12, \"h\": 10}, \"panelIndex\": \"4\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_4\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"5\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_5\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 16, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"6\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_6\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 32, \"y\": 10, \"w\": 16, \"h\": 14}, \"panelIndex\": \"7\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_7\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 24, \"w\": 24, \"h\": 15}, \"panelIndex\": \"8\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_8\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 24, \"w\": 24, \"h\": 15}, \"panelIndex\": \"9\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_9\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 39, \"w\": 24, \"h\": 18}, \"panelIndex\": \"10\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_10\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 39, \"w\": 24, \"h\": 18}, \"panelIndex\": \"11\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_11\"}, {\"version\": \"8.19.14\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 57, \"w\": 48, \"h\": 22}, \"panelIndex\": \"12\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_12\"}]", "refreshInterval": {"pause": false, "value": 10000}, "timeRestore": true, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"query\": {\"language\": \"kuery\", \"query\": \"\"}, \"filter\": []}"}, "timeFrom": "now-2h", "timeTo": "now"}, "references": [{"type": "visualization", "id": "viz-op-events-recent", "name": "panel_1"}, {"type": "visualization", "id": "viz-op-risk-events", "name": "panel_2"}, {"type": "visualization", "id": "viz-op-ssh-failures", "name": "panel_3"}, {"type": "visualization", "id": "viz-op-firewall-blocks", "name": "panel_4"}, {"type": "visualization", "id": "viz-exec-severity-distribution", "name": "panel_5"}, {"type": "visualization", "id": "viz-op-actions", "name": "panel_6"}, {"type": "visualization", "id": "viz-op-web-indicators", "name": "panel_7"}, {"type": "visualization", "id": "viz-op-risk-trend", "name": "panel_8"}, {"type": "visualization", "id": "viz-op-blocked-ports", "name": "panel_9"}, {"type": "visualization", "id": "viz-exec-risk-ips", "name": "panel_10"}, {"type": "visualization", "id": "viz-op-top-auth-users", "name": "panel_11"}, {"type": "visualization", "id": "viz-op-investigation-events", "name": "panel_12"}]} diff --git a/setup/generate-normal-logs.py b/setup/generate-normal-logs.py new file mode 100644 index 0000000..9a90152 --- /dev/null +++ b/setup/generate-normal-logs.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +generate-normal-logs.py +Genera logs LEGITIMOS (no ataques) para las 4 fuentes del SIEM. +Cumple R8.2: syslog, auth.log, nginx/apache, kubernetes. + +Escribe directamente en los archivos que Filebeat monitorea. + +Uso: + python3 generate-normal-logs.py --source all --count 30 + python3 generate-normal-logs.py --source nginx --count 50 + python3 generate-normal-logs.py --source auth +""" + +import argparse +import json +import random +from datetime import datetime, timezone, timedelta +from pathlib import Path + +BASE_LOG_DIR = Path(__file__).resolve().parent.parent / "logs" +BOGOTA_TZ = timezone(timedelta(hours=-5)) + +INTERNAL_IPS = ["10.0.0.5", "192.168.1.20", "172.16.0.10"] +EXTERNAL_IPS = [ + "185.220.101.34", "89.248.167.131", "194.165.16.11", + "45.33.32.156", "104.236.246.58", "162.243.157.229", +] +USERS = ["ubuntu", "deploy", "admin", "www-data", "gitlab"] +PATHS_OK = ["/", "/index.html", "/api/v1/status", "/api/v1/users", "/about", + "/api/health", "/login", "/css/style.css", "/js/app.js", "/favicon.ico"] + + +def ts_syslog(): + return datetime.now(BOGOTA_TZ).strftime("%b %d %H:%M:%S") + + +def ts_nginx(): + return datetime.now(timezone.utc).strftime("%d/%b/%Y:%H:%M:%S +0000") + + +def ts_iso(): + return datetime.now(timezone.utc).isoformat() + + +def write_logs(lines, path): + Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + for line in lines: + f.write(line + "\n") + print(f" -> {len(lines)} lineas escritas en {path}") + + +# --------------------------------------------------------------------------- +# FUENTE 1: SYSLOG - mensajes normales del sistema +# --------------------------------------------------------------------------- +def gen_syslog(count=20): + """Genera entradas de syslog del sistema (eventos normales).""" + templates = [ + "{ts} {host} systemd[1]: Started Session {sid} of user {user}.", + "{ts} {host} systemd[1]: Stopped Session {sid} of user {user}.", + "{ts} {host} cron[{pid}]: ({user}) CMD (/usr/bin/python3 /opt/monitor.py)", + "{ts} {host} kernel: EXT4-fs (sda1): mounted filesystem with writeback timezone.", + "{ts} {host} systemd-logind[{pid}]: New session {sid} of user {user}.", + "{ts} {host} CRON[{pid}]: (root) CMD (test -x /usr/sbin/anacron || exit 0)", + "{ts} {host} NetworkManager[{pid}]: device (eth0): state change: ip-config -> activated", + "{ts} {host} rsyslogd[{pid}]: [origin software=\"rsyslogd\"] start", + "{ts} {host} ntpd[{pid}]: time sync resumed, clock offset 0.003 sec", + "{ts} {host} sshd[{pid}]: Server listening on 0.0.0.0 port 22.", + ] + lines = [] + for _ in range(count): + tpl = random.choice(templates) + lines.append(tpl.format( + ts = ts_syslog(), + host = random.choice(["web-01", "db-01", "app-01"]), + user = random.choice(USERS), + pid = random.randint(1000, 9999), + sid = random.randint(1, 100), + )) + return lines + + +# --------------------------------------------------------------------------- +# FUENTE 2: AUTH.LOG - autenticaciones exitosas y operaciones normales +# --------------------------------------------------------------------------- +def gen_auth_normal(count=10): + """Genera eventos de autenticacion normales (exitosos).""" + lines = [] + for _ in range(count): + ip = random.choice(INTERNAL_IPS) + user = random.choice(["ubuntu", "deploy", "admin"]) + pid = random.randint(1000, 9999) + event_type = random.choice(["accepted", "session_open", "sudo", "session_close"]) + + if event_type == "accepted": + lines.append( + f"{ts_syslog()} web-01 sshd[{pid}]: " + f"Accepted publickey for {user} from {ip} port " + f"{random.randint(40000,65000)} ssh2: RSA SHA256:def456" + ) + + elif event_type == "session_open": + lines.append( + f"{ts_syslog()} web-01 sshd[{pid}]: " + f"pam_unix(sshd:session): session opened for user {user} by (uid=0)" + ) + + elif event_type == "sudo": + cmd = random.choice(["/bin/bash", "/usr/bin/git pull", "/bin/systemctl restart nginx", + "/usr/bin/apt update", "/bin/journalctl -xe", "/usr/sbin/ufw status"]) + lines.append( + f"{ts_syslog()} web-01 sudo: {user} : TTY=pts/0 ; " + f"PWD=/home/{user} ; USER=root ; COMMAND={cmd}" + ) + + else: + lines.append( + f"{ts_syslog()} web-01 sshd[{pid}]: " + f"pam_unix(sshd:session): session closed for user {user}" + ) + + return lines + + +# --------------------------------------------------------------------------- +# AUTH - Logins fuera de horario laboral (R8.4 - Login SSH Fuera de Horario) +# Genera eventos a las 02:30 GMT-5 para que el pipeline Logstash +# los etiquete con siem.after_hours_login=true (regla: hour < 9 || hour >= 18) +# --------------------------------------------------------------------------- +def ts_syslog_off_hours(): + """Timestamp a las 02:30 GMT-5 fuera del horario laboral 09-18.""" + now = datetime.now(BOGOTA_TZ) + off_hours = now.replace(hour=2, minute=30, second=0, microsecond=0) + return off_hours.strftime("%b %d %H:%M:%S") + + +def gen_auth_after_hours(count=5): + """Genera logins SSH exitosos con timestamp fuera del horario laboral.""" + ips = EXTERNAL_IPS[:3] + lines = [] + for _ in range(count): + ip = random.choice(ips) + user = random.choice(["deploy", "admin", "root"]) + pid = random.randint(1000, 9999) + ts = ts_syslog_off_hours() + lines.append( + f"{ts} web-01 sshd[{pid}]: " + f"Accepted password for {user} from {ip} port " + f"{random.randint(40000,65000)} ssh2" + ) + return lines + + +# --------------------------------------------------------------------------- +# FUENTE 3: NGINX - trafico web normal +# --------------------------------------------------------------------------- +def gen_nginx_normal(count=30): + """Genera trafico nginx normal (codigos 200, 301, 304, 404 aislados).""" + codes = [200, 200, 200, 200, 301, 304, 200, 200, 200, 404] + agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36", + "curl/7.68.0", + "python-requests/2.28.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", + ] + lines = [] + for _ in range(count): + ip = random.choice(INTERNAL_IPS + ["104.236.246.58", "45.33.32.156"]) + path = random.choice(PATHS_OK) + code = random.choice(codes) + size = random.randint(100, 50000) + ua = random.choice(agents) + lines.append( + f'{ip} - - [{ts_nginx()}] ' + f'"GET {path} HTTP/1.1" {code} {size} ' + f'"-" "{ua}"' + ) + return lines + + +# --------------------------------------------------------------------------- +# FUENTE 4: KUBERNETES - logs de contenedores +# --------------------------------------------------------------------------- +def gen_k8s_logs(count=15): + """Genera logs de contenedores Kubernetes (eventos normales).""" + pods = [ + ("frontend-7d9b8c-xk2p9", "frontend", "default"), + ("backend-api-5f6b-mn3q1", "api", "default"), + ("db-primary-0", "postgres", "database"), + ("redis-master-0", "redis", "cache"), + ("nginx-ingress-abc12", "nginx", "kube-system"), + ] + lines = [] + for _ in range(count): + pod, ctr, ns = random.choice(pods) + msg_type = random.choice(["info", "info", "info", "warn"]) + messages = { + "info": f"Request processed in {random.randint(10,500)}ms", + "warn": f"Connection pool at {random.randint(70,85)}% capacity", + } + lines.append(json.dumps({ + "time": ts_iso(), + "level": msg_type, + "message": messages[msg_type], + "kubernetes": { + "pod": {"name": pod}, + "namespace": ns, + "container": {"name": ctr}, + } + })) + return lines + + +# --------------------------------------------------------------------------- +# MAIN +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description="Generador de logs normales SIEM (R8.2)") + parser.add_argument("--source", choices=["all", "syslog", "auth", "nginx", "k8s"], + default="all", help="Fuente de logs a generar") + parser.add_argument("--count", type=int, default=30, + help="Numero de entradas de log por fuente") + parser.add_argument("--syslog-path", default=str(BASE_LOG_DIR / "syslog")) + parser.add_argument("--auth-path", default=str(BASE_LOG_DIR / "auth.log")) + parser.add_argument("--nginx-path", default=str(BASE_LOG_DIR / "nginx" / "access.log")) + parser.add_argument("--k8s-path", default=str(BASE_LOG_DIR / "containers" / "test-pod.log")) + args = parser.parse_args() + + print(f"\nGenerador de logs normales SIEM - {ts_iso()}\n") + + if args.source in ("all", "syslog"): + print("Generando syslog normal...") + write_logs(gen_syslog(args.count), args.syslog_path) + + if args.source in ("all", "auth"): + print("Generando auth.log normal...") + write_logs(gen_auth_normal(max(args.count // 2, 5)), args.auth_path) + print("Generando auth.log (fuera de horario laboral - 02:30 GMT-5)...") + write_logs(gen_auth_after_hours(5), args.auth_path) + + if args.source in ("all", "nginx"): + print("Generando nginx access normal...") + write_logs(gen_nginx_normal(args.count), args.nginx_path) + + if args.source in ("all", "k8s"): + print("Generando logs Kubernetes normales...") + write_logs(gen_k8s_logs(max(args.count // 2, 10)), args.k8s_path) + + print("\nLogs normales generados. Esperar ~15s para que Filebeat los procese.") + + +if __name__ == "__main__": + main() diff --git a/setup/generate-test-logs.py b/setup/generate-test-logs.py new file mode 100644 index 0000000..e75a735 --- /dev/null +++ b/setup/generate-test-logs.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +generate-test-logs.py +Genera logs de ATAQUE simulados para el SIEM. +Cumple R8.5: brute force SSH, 404 flood, SQL injection, port scan (nmap). + +Escribe directamente en los archivos que Filebeat monitorea. + +Uso: + python3 generate-test-logs.py --attack all_attacks + python3 generate-test-logs.py --attack brute_force --count 50 + python3 generate-test-logs.py --attack port_scan +""" + +import argparse +import random +import time +from datetime import datetime, timezone, timedelta +from pathlib import Path + +BASE_LOG_DIR = Path(__file__).resolve().parent.parent / "logs" +BOGOTA_TZ = timezone(timedelta(hours=-5)) + +EXTERNAL_IPS = [ + "185.220.101.34", "89.248.167.131", "194.165.16.11", + "45.33.32.156", "198.199.101.45", "104.236.246.58", + "162.243.157.229","159.89.214.31", "167.99.197.32", +] +USERS = ["root", "admin", "ubuntu", "deploy", "postgres", "www-data"] +PATHS_SENS = ["/admin", "/.env", "/wp-admin", "/phpmyadmin", "/config.php"] +PATHS_SQLI = [ + "/api/search?q=1' OR '1'='1", + "/users?id=1 UNION SELECT username,password FROM users--", + "/api/data?filter=1;DROP TABLE users;--", +] + + +def ts_syslog(): + return datetime.now(BOGOTA_TZ).strftime("%b %d %H:%M:%S") + + +def ts_nginx(): + return datetime.now(timezone.utc).strftime("%d/%b/%Y:%H:%M:%S +0000") + + +def write_logs(lines, path): + Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + for line in lines: + f.write(line + "\n") + print(f" -> {len(lines)} lineas escritas en {path}") + + +# --------------------------------------------------------------------------- +# ATAQUE 1: Brute Force SSH +# --------------------------------------------------------------------------- +def gen_auth_brute_force(target_ip=None, count=30): + """Genera un ataque de brute force SSH (R8.5 - escenario 1).""" + ip = target_ip or random.choice(EXTERNAL_IPS) + users = ["root", "admin", "ubuntu", "user", "test", "postgres"] + lines = [] + print(f" [brute_force] IP atacante: {ip}, {count} intentos") + for i in range(count): + user = random.choice(users) + pid = random.randint(1000, 9999) + lines.append( + f"{ts_syslog()} web-01 sshd[{pid}]: " + f"Failed password for {'invalid user ' if random.random() > 0.5 else ''}" + f"{user} from {ip} port {random.randint(40000,65000)} ssh2" + ) + time.sleep(0.05) + return lines + + +# --------------------------------------------------------------------------- +# ATAQUE 2: 404 Flood (enumeracion web) +# --------------------------------------------------------------------------- +def gen_nginx_404_flood(source_ip=None, count=20): + """Genera flood de errores 404 (R8.5 - escenario 2).""" + ip = source_ip or random.choice(EXTERNAL_IPS) + paths = [f"/wp-includes/{i}.php" for i in range(50)] + \ + ["/.git/config", "/.env", "/backup.zip", "/admin/config"] + lines = [] + print(f" [404_flood] IP atacante: {ip}, {count} peticiones") + for _ in range(count): + path = random.choice(paths) + lines.append( + f'{ip} - - [{ts_nginx()}] ' + f'"GET {path} HTTP/1.1" 404 162 ' + f'"-" "python-requests/2.28.0"' + ) + time.sleep(0.02) + return lines + + +# --------------------------------------------------------------------------- +# ATAQUE 3: SQL Injection +# --------------------------------------------------------------------------- +def gen_nginx_sqli(source_ip=None, count=10): + """Genera intentos de inyeccion SQL en logs nginx (R8.5 - escenario 3).""" + ip = source_ip or random.choice(EXTERNAL_IPS) + lines = [] + print(f" [sqli] IP atacante: {ip}, {count} intentos") + for _ in range(count): + path = random.choice(PATHS_SQLI) + lines.append( + f'{ip} - - [{ts_nginx()}] ' + f'"GET {path} HTTP/1.1" 200 1024 ' + f'"-" "sqlmap/1.7.2#stable (https://sqlmap.org)"' + ) + time.sleep(0.1) + return lines + + +# --------------------------------------------------------------------------- +# ATAQUE 4: Port Scan (nmap) +# --------------------------------------------------------------------------- +def gen_port_scan_nmap(target_ip=None, count=30): + """Genera eventos UFW BLOCK simulando un escaneo nmap SYN a multiples puertos + (R8.5 - escenario nmap). Dispara la regla 'SIEM - Escaneo de Puertos Detectado' + cuando una misma IP contacta 20+ puertos distintos en 30s.""" + ip = target_ip or random.choice(EXTERNAL_IPS) + ports = [22, 80, 443, 3306, 5432, 8080, 8443, 9200, 5601, 5044, + 6379, 27017, 1433, 1521, 11211, 25, 53, 161, 389, 636, + 993, 995, 2082, 2083, 8888, 9090, 3000, 5000, 9000, 10000] + host = random.choice(["web-01", "app-01", "db-01"]) + lines = [] + print(f" [port_scan_nmap] IP atacante: {ip}, {count} puertos escaneados") + for i in range(min(count, len(ports))): + port = ports[i] + d_ip = f"10.0.0.{random.randint(1, 20)}" + pid = random.randint(1000, 9999) + lines.append( + f"{ts_syslog()} {host} kernel: [{random.randint(100000, 999999)}.{random.randint(100, 999)}] " + f"[UFW BLOCK] IN=eth0 OUT= SRC={ip} DST={d_ip} " + f"LEN={random.randint(40, 120)} TTL={random.randint(40, 128)} " + f"PROTO=TCP SPT={random.randint(40000, 65000)} DPT={port}" + ) + time.sleep(0.08) + return lines + + +# --------------------------------------------------------------------------- +# Complementario: Acceso a rutas sensibles (se genera con sqli) +# --------------------------------------------------------------------------- +def gen_nginx_sensitive_paths(source_ip=None, count=8): + """Genera acceso a rutas sensibles.""" + ip = source_ip or random.choice(EXTERNAL_IPS) + lines = [] + for path in PATHS_SENS[:count]: + lines.append( + f'{ip} - - [{ts_nginx()}] ' + f'"GET {path} HTTP/1.1" 403 287 ' + f'"-" "Mozilla/5.0"' + ) + return lines + + +# --------------------------------------------------------------------------- +# MAIN +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description="Generador de logs de ataque SIEM (R8.5)") + parser.add_argument("--attack", choices=["brute_force", "404_flood", "sqli", "port_scan", "all_attacks"], + default="all_attacks", help="Tipo de ataque a simular") + parser.add_argument("--count", type=int, default=30, + help="Numero de entradas de log por ataque") + parser.add_argument("--auth-path", default=str(BASE_LOG_DIR / "auth.log")) + parser.add_argument("--nginx-path", default=str(BASE_LOG_DIR / "nginx" / "access.log")) + parser.add_argument("--syslog-path", default=str(BASE_LOG_DIR / "syslog")) + args = parser.parse_args() + + attacker_ip = random.choice(EXTERNAL_IPS) + print(f"\nGenerador de ataques SIEM - {datetime.now(timezone.utc).isoformat()}") + print(f"IP atacante para simulaciones: {attacker_ip}\n") + + if args.attack in ("brute_force", "all_attacks"): + print("Simulando brute force SSH...") + write_logs(gen_auth_brute_force(attacker_ip, max(args.count, 35)), args.auth_path) + + if args.attack in ("404_flood", "all_attacks"): + print("Simulando 404 flood...") + write_logs(gen_nginx_404_flood(attacker_ip, max(args.count, 25)), args.nginx_path) + + if args.attack in ("sqli", "all_attacks"): + print("Simulando inyeccion SQL...") + write_logs(gen_nginx_sqli(attacker_ip, 12), args.nginx_path) + write_logs(gen_nginx_sensitive_paths(attacker_ip), args.nginx_path) + + if args.attack in ("port_scan", "all_attacks"): + print("Simulando escaneo de puertos (nmap)...") + write_logs(gen_port_scan_nmap(attacker_ip, 30), args.syslog_path) + + print("\nAtaques generados. Esperar ~30s para que Filebeat los procese.") + print("Verificar en Kibana -> Security -> Alerts") + + +if __name__ == "__main__": + main() diff --git a/setup/geoip-pipeline.json b/setup/geoip-pipeline.json new file mode 100644 index 0000000..92ca79b --- /dev/null +++ b/setup/geoip-pipeline.json @@ -0,0 +1,11 @@ +{ + "processors": [ + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + } + ] +} diff --git a/setup/ilm-policy.json b/setup/ilm-policy.json new file mode 100644 index 0000000..c7b4c52 --- /dev/null +++ b/setup/ilm-policy.json @@ -0,0 +1,50 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "5gb", + "max_age": "1d", + "max_docs": 10000000 + }, + "set_priority": { + "priority": 100 + } + } + }, + "warm": { + "min_age": "7d", + "actions": { + "shrink": { + "number_of_shards": 1 + }, + "forcemerge": { + "max_num_segments": 1 + }, + "set_priority": { + "priority": 50 + }, + "readonly": {} + } + }, + "cold": { + "min_age": "15d", + "actions": { + "set_priority": { + "priority": 0 + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": { + "delete_searchable_snapshot": true + } + } + } + } + } +} diff --git a/setup/import-dashboards.py b/setup/import-dashboards.py new file mode 100644 index 0000000..2265d38 --- /dev/null +++ b/setup/import-dashboards.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Importa dashboards a Kibana desde NDJSON.""" + +import os +import sys +import time +from pathlib import Path + +try: + import requests + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + print("ERROR: pip install requests") + sys.exit(1) + +KIBANA_URL = os.getenv("KIBANA_HOST", "https://localhost:5601") +ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic") +ELASTIC_PASS = os.getenv("ELASTIC_PASSWORD", "SiemElastic2026!") +CACERT = os.getenv("CACERT", "./setup/certs/ca/ca.crt") +FILE = Path(__file__).resolve().parent / "dashboards" / "executive-operational-dashboards.ndjson" + +G = "\033[92m" +R = "\033[91m" +Y = "\033[93m" +W = "\033[1m" +X = "\033[0m" + + +def kibana_session(): + client = requests.Session() + client.auth = (ELASTIC_USER, ELASTIC_PASS) + client.headers = {"kbn-xsrf": "true"} + client.verify = CACERT if Path(CACERT).exists() else False + return client + + +def wait_for_kibana(client, timeout=180): + print(f"\n{W}Esperando a que Kibana esté disponible...{X}") + deadline = time.time() + timeout + while time.time() < deadline: + try: + response = client.get(f"{KIBANA_URL}/api/status", timeout=5) + if response.status_code == 200: + print(f" {G}✓{X} Kibana disponible") + return True + except Exception: + pass + time.sleep(2) + print(f" {R}✗{X} Kibana no respondió en {timeout}s") + return False + + +def import_dashboards(): + print(f"\n{W}=== Importando Dashboards a Kibana ==={X}") + client = kibana_session() + + if not wait_for_kibana(client): + sys.exit(1) + + if not FILE.exists(): + print(f"{R}No se encontró: {FILE}{X}") + sys.exit(1) + + print(f" Importando desde: {W}{FILE.name}{X}") + + with FILE.open("rb") as file_handle: + response = client.post( + f"{KIBANA_URL}/api/saved_objects/_import", + params={"overwrite": "true"}, + files={"file": (FILE.name, file_handle, "application/x-ndjson")}, + timeout=30, + ) + + if response.status_code != 200: + print(f" {R}✗{X} Error {response.status_code}") + print(response.text[:500]) + sys.exit(1) + + payload = response.json() + imported = payload.get("success_count", payload.get("successCount", 0)) + success_results = payload.get("successResults", []) + dashboard_count = sum(1 for item in success_results if item.get("type") == "dashboard") + errors = payload.get("errors", payload.get("errorResults", [])) + + if errors: + print(f" {Y}⚠{X} Importados {dashboard_count} dashboards y {imported} objetos con {len(errors)} errores") + for error in errors[:5]: + print(f" - {error.get('type', 'N/A')}: {error.get('id', 'N/A')}") + else: + print(f" {G}✓{X} {dashboard_count} dashboards importados") + + +if __name__ == "__main__": + import_dashboards() diff --git a/setup/import-rules.py b/setup/import-rules.py new file mode 100644 index 0000000..a095c4a --- /dev/null +++ b/setup/import-rules.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +""" +import-rules.py +Importa las 5 reglas de detección SIEM a Kibana usando el endpoint de importación. +También puede probar cada regla generando eventos sintéticos. + +Uso: + pip install requests + python3 setup/import-rules.py --action import + python3 setup/import-rules.py --action test + python3 setup/import-rules.py --action status +""" + +import argparse +import json +import time +import sys +import os +import random +from datetime import datetime, timezone, timedelta +from pathlib import Path + +try: + import requests + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + print("ERROR: pip install requests") + sys.exit(1) + +# ─── Configuración ──────────────────────────────────────────────────────────── +KIBANA_URL = os.getenv("KIBANA_HOST", "https://localhost:5601") +ES_URL = os.getenv("ELASTIC_HOSTS", "https://localhost:9200") +ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic") +ELASTIC_PASS = os.getenv("ELASTIC_PASSWORD","SiemElastic2026!") +CACERT = os.getenv("CACERT", "./setup/certs/ca/ca.crt") +RULES_FILE = Path(__file__).resolve().parent.parent / "rules" / "all-rules.ndjson" + +# Colores +G = "\033[92m"; R = "\033[91m"; Y = "\033[93m" +B = "\033[94m"; W = "\033[1m"; X = "\033[0m" + +def kibana_session(): + s = requests.Session() + s.auth = (ELASTIC_USER, ELASTIC_PASS) + s.headers = { + "kbn-xsrf": "true", + } + s.verify = CACERT if Path(CACERT).exists() else False + return s + +def es_session(): + s = requests.Session() + s.auth = (ELASTIC_USER, ELASTIC_PASS) + s.verify = CACERT + return s + +# ─── Esperar a que Kibana esté disponible ───────────────────────────────── +def wait_for_kibana(session, kibana_url, timeout_seconds=180): + print(f"\n{W}=== Esperando a que Kibana esté disponible en {kibana_url} ==={X}") + deadline = time.time() + timeout_seconds + while time.time() < deadline: + try: + response = session.get(kibana_url, timeout=10) + if response.status_code in (200, 302): + print(f" {G}Kibana disponible{X}") + return True + except requests.exceptions.RequestException: + pass + print(" esperando...", end="\r", flush=True) + time.sleep(5) + print(f"\n{R}ERROR: Kibana no respondió en {timeout_seconds} segundos{X}") + return False + +def security_solution_available(session): + try: + response = session.get(f"{KIBANA_URL}/api/status", timeout=10) + if response.status_code != 200: + return False + data = response.json() + plugins = data.get("status", {}).get("plugins", {}) + return any(name in plugins for name in ("securitySolution", "security_solution_ess", "security_solution_serverless")) + except requests.exceptions.RequestException: + return False + +# ─── IMPORTAR REGLAS ────────────────────────────────────────────────────────── +def import_rules(): + print(f"\n{W}=== Importando reglas a Kibana SIEM ==={X}") + session = kibana_session() + + if not wait_for_kibana(session, KIBANA_URL): + sys.exit(1) + + if not security_solution_available(session): + print(f"{Y}Security Solution APIs not available in this Kibana build; skipping rule import.{X}") + return + + if not RULES_FILE.exists(): + print(f"{R}No se encontró el archivo {RULES_FILE}{X}") + sys.exit(1) + + print(f"\n Importando desde: {B}{RULES_FILE}{X}") + with RULES_FILE.open("rb") as rule_file: + response = session.post( + f"{KIBANA_URL}/api/detection_engine/rules/_import", + params={"overwrite": "true"}, + files={"file": (RULES_FILE.name, rule_file, "application/x-ndjson")}, + timeout=30, + ) + + if response.status_code != 200: + print(f" {R}✗{X} Error {response.status_code}: {response.text[:300]}") + sys.exit(1) + + payload = response.json() + imported = payload.get("success_count", 0) + total = payload.get("rules_count", 0) + if not payload.get("success"): + print(f" {R}✗{X} Importación incompleta: {imported}/{total}") + print(json.dumps(payload, indent=2, ensure_ascii=False)) + sys.exit(1) + + print(f" {G}✓{X} Importadas {imported}/{total} reglas") + +# ─── ESTADO DE REGLAS ───────────────────────────────────────────────────────── +def check_status(): + print(f"\n{W}=== Estado de las reglas SIEM ==={X}") + session = kibana_session() + if not security_solution_available(session): + print(f"{Y}Security Solution APIs not available in this Kibana build; no rule status to show.{X}") + return + r = session.get( + f"{KIBANA_URL}/api/detection_engine/rules/_find?per_page=20&filter=alert.attributes.tags:%22SIEM%22", + verify=False, timeout=30, + ) + if r.status_code != 200: + print(f"{R}Error conectando a Kibana: {r.status_code}{X}") + return + + data = r.json() + rules = data.get("data", []) + print(f" Reglas SIEM encontradas: {len(rules)}\n") + + headers = ["Nombre", "Habilitada", "Severidad", "Último run", "Alertas totales"] + print(f" {'Nombre':<45} {'Habilitada':<12} {'Sev':<8} {'Alertas'}") + print(f" {'-'*45} {'-'*12} {'-'*8} {'-'*8}") + + for rule in rules: + name = rule.get("name", "?")[:44] + enabled = f"{G}Sí{X}" if rule.get("enabled") else f"{R}No{X}" + sev = rule.get("severity", "?") + sev_col = G if sev == "low" else (Y if sev == "medium" else R) + alerts = rule.get("execution_summary", {}).get("last_execution", {}).get("metrics", {}).get("total_alerts_created", "?") + + print(f" {name:<45} {enabled:<12} {sev_col}{sev:<8}{X} {alerts}") + +# ─── GENERAR EVENTOS DE PRUEBA ──────────────────────────────────────────────── +def generate_test_events(): + """ + Inserta documentos directamente en Elasticsearch para disparar cada regla. + Útil para demostrar R8.4 sin necesidad de ataques reales. + """ + print(f"\n{W}=== Generando eventos de prueba para las 5 reglas ==={X}") + es = es_session() + now = datetime.now(timezone.utc) + attacker_ip = "185.220.101.34" + + tests = [ + # ── Regla 1: Brute Force SSH ───────────────────────────────────────── + { + "rule": "Brute Force SSH", + "index": "logs-auth", + "count": 8, + "doc_template": { + "@timestamp": None, # se rellena en el bucle + "event.dataset": "system.auth", + "event.outcome": "failure", + "event.action": "authentication_failure", + "event.kind": "event", + "event.category": "authentication", + "process.name": "sshd", + "host.name": "web-01", + "source.ip": attacker_ip, + "user.name": "root", + "tags": ["auth_failure", "ssh_failure"], + "siem.brute_force_attempt": True, + "siem.severity": "high", + "message": f"Failed password for root from {attacker_ip} port 54321 ssh2", + } + }, + # ── Regla 2: Port Scan ─────────────────────────────────────────────── + { + "rule": "Port Scan", + "index": "logs-syslog", + "count": 25, + "doc_template": { + "@timestamp": None, + "event.dataset": "system.syslog", + "event.category": "network", + "event.action": "firewall_block", + "event.kind": "event", + "event.outcome": "failure", + "network.transport": "tcp", + "host.name": "web-01", + "source.ip": attacker_ip, + "destination.ip": "10.0.0.1", + "tags": ["firewall_block"], + "siem.severity": "medium", + "message": f"[UFW BLOCK] SRC={attacker_ip} PROTO=TCP", + }, + "vary_field": "destination.port", + "vary_values": list(range(20, 45)) + }, + # ── Regla 3: Múltiples 404 ─────────────────────────────────────────── + { + "rule": "Múltiples 404", + "index": "logs-nginx", + "count": 20, + "doc_template": { + "@timestamp": None, + "event.dataset": "nginx.access", + "event.category": "web", + "event.type": "denied", + "event.kind": "event", + "http.request.method": "GET", + "http.response.status_code": 404, + "http.response.body.bytes": 162, + "http.version": "1.1", + "source.ip": attacker_ip, + "user_agent.original": "gobuster/3.1.0", + "tags": ["http_404"], + "siem.severity": "medium", + "message": f"{attacker_ip} - - \"GET /wp-admin HTTP/1.1\" 404 162", + }, + "vary_field": "url.path", + "vary_values": [ + "/wp-admin", "/.env", "/config.php", "/backup.zip", + "/.git/config", "/admin", "/phpmyadmin", "/setup.php", + "/install.php", "/wp-login.php", "/administrator", + "/db.sql", "/dump.sql", "/admin/login", "/console", + "/actuator", "/.aws/credentials", "/server-info", + "/server-status", "/robots.txt", "/sitemap.xml" + ] + }, + # ── Regla 4: Login fuera de horario ────────────────────────────────── + { + "rule": "Login Fuera de Horario", + "index": "logs-auth", + "count": 1, + "doc_template": { + "@timestamp": (now.replace(hour=2, minute=30)).isoformat(), + "event.dataset": "system.auth", + "event.outcome": "success", + "event.action": "authentication_success", + "event.kind": "event", + "event.category": "authentication", + "process.name": "sshd", + "host.name": "web-01", + "source.ip": attacker_ip, + "source.geo.country_iso_code": "RU", + "source.geo.country_name": "Russia", + "user.name": "deploy", + "auth.method": "password", + "siem.after_hours_login": True, + "siem.severity": "medium", + "tags": ["auth_success", "ssh_login", "after_hours_login"], + "message": f"Accepted password for deploy from {attacker_ip} port 54321 ssh2", + } + }, + # ── Regla 5: Rutas sensibles ───────────────────────────────────────── + { + "rule": "Acceso a Rutas Sensibles", + "index": "logs-nginx", + "count": 5, + "doc_template": { + "@timestamp": None, + "event.dataset": "nginx.access", + "event.category": "web", + "event.kind": "event", + "http.request.method": "GET", + "http.response.status_code": 200, + "http.response.body.bytes": 4096, + "source.ip": attacker_ip, + "user_agent.original": "curl/7.68.0", + "siem.sensitive_path": True, + "siem.severity": "high", + "tags": ["sensitive_path_access"], + "message": f"{attacker_ip} - - \"GET /.env HTTP/1.1\" 200 4096", + }, + "vary_field": "url.path", + "vary_values": ["/.env", "/admin", "/.git/config", "/phpmyadmin", "/backup.zip"] + }, + ] + + total_inserted = 0 + + for test in tests: + rule_name = test["rule"] + index = f"{test['index']}-{now.strftime('%Y.%m.%d')}" + count = test["count"] + template = test["doc_template"].copy() + vary = test.get("vary_field") + values = test.get("vary_values", []) + + print(f"\n {B}── {rule_name}{X} → {index}") + + inserted = 0 + for i in range(count): + doc = template.copy() + # Timestamp dinámico (distribuido en últimos 45 segundos) + if doc.get("@timestamp") is None or isinstance(doc["@timestamp"], type(None)): + doc["@timestamp"] = ( + now - timedelta(seconds=random.randint(0, 45)) + ).isoformat() + # Variar campo si aplica + if vary and values: + doc[vary] = values[i % len(values)] + + r = es.post( + f"{ES_URL}/{index}/_doc", + json=doc, + timeout=10, + ) + if r.status_code in (200, 201): + inserted += 1 + else: + print(f" {Y}⚠{X} Doc {i}: {r.status_code} — {r.text[:100]}") + + print(f" {G}✓{X} {inserted}/{count} documentos insertados") + total_inserted += inserted + + print(f"\n {W}Total insertado: {total_inserted} documentos{X}") + print(f"\n {Y}Esperar ~60 segundos y verificar alertas en:{X}") + print(f" Kibana → Security → Alerts") + print(f" O ejecutar: python3 setup/import-rules.py --action status") + +# ─── MAIN ───────────────────────────────────────────────────────────────────── +def main(): + global KIBANA_URL, ES_URL, ELASTIC_USER, ELASTIC_PASS, CACERT + + parser = argparse.ArgumentParser( + description="Gestión de reglas SIEM en Kibana" + ) + parser.add_argument( + "--action", + choices=["import", "test", "status"], + default="import", + help="import: carga reglas | test: inserta eventos | status: muestra estado" + ) + parser.add_argument("--kibana", default=KIBANA_URL, help="URL de Kibana") + parser.add_argument("--es", default=ES_URL, help="URL de Elasticsearch") + parser.add_argument("--user", default=ELASTIC_USER) + parser.add_argument("--password", default=ELASTIC_PASS) + parser.add_argument("--cacert", default=CACERT) + args = parser.parse_args() + + print(f"{W}SIEM Rules Manager{X} — {datetime.now(timezone(timedelta(hours=-5))).strftime('%Y-%m-%d %H:%M UTC-5')}") + print(f"Kibana: {args.kibana} | ES: {args.es}") + KIBANA_URL = args.kibana + ES_URL = args.es + ELASTIC_USER = args.user + ELASTIC_PASS = args.password + CACERT = args.cacert + + if args.action == "import": + import_rules() + elif args.action == "test": + generate_test_events() + elif args.action == "status": + check_status() + +if __name__ == "__main__": + main() diff --git a/setup/logs-mapping.json b/setup/logs-mapping.json new file mode 100644 index 0000000..d1500a1 --- /dev/null +++ b/setup/logs-mapping.json @@ -0,0 +1,28 @@ +{ + "index_patterns": ["logs-*"], + "template": { + "mappings": { + "properties": { + "source": { + "properties": { + "geo": { + "properties": { + "location": { "type": "geo_point" } + } + } + } + }, + "destination": { + "properties": { + "geo": { + "properties": { + "location": { "type": "geo_point" } + } + } + } + } + } + } + }, + "priority": 500 +} diff --git a/setup/orchestrate-logs.py b/setup/orchestrate-logs.py new file mode 100644 index 0000000..e7a66a5 --- /dev/null +++ b/setup/orchestrate-logs.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +""" +orchestrate-logs.py +Orquestador de generacion de logs para el SIEM. +Cumple R8.5: coordina logs de ataque (generate-test-logs.py) y +logs normales (generate-normal-logs.py) en 3 modos. + +Uso: + python3 orchestrate-logs.py --mode random # mix aleatorio de normales + ataques + python3 orchestrate-logs.py --mode attacks # solo ataques + python3 orchestrate-logs.py --mode normal # solo logs normales +""" + +import argparse +import random +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent + + +def call_attack_script(attack_type="all_attacks", count=30): + """Ejecuta generate-test-logs.py con el tipo de ataque indicado.""" + script = SCRIPT_DIR / "generate-test-logs.py" + print(f" >> Llamando: {script.name} --attack {attack_type}") + result = subprocess.run( + [sys.executable, str(script), "--attack", attack_type, "--count", str(count)], + capture_output=False, + ) + return result.returncode == 0 + + +def call_normal_script(source="all", count=30): + """Ejecuta generate-normal-logs.py con la fuente indicada.""" + script = SCRIPT_DIR / "generate-normal-logs.py" + print(f" >> Llamando: {script.name} --source {source}") + result = subprocess.run( + [sys.executable, str(script), "--source", source, "--count", str(count)], + capture_output=False, + ) + return result.returncode == 0 + + +# --------------------------------------------------------------------------- +# MODO 1: Logs aleatorios (mix de normales + ataques) +# --------------------------------------------------------------------------- +def generate_random_mix(): + """Funcion 1: llama aleatoriamente a los 2 scripts para generar un mix.""" + print("\n=== MODO RANDOM: Mix aleatorio de logs normales y ataques ===\n") + operations = [] + + for _ in range(random.randint(2, 4)): + operations.append(("normal", random.choice(["all", "syslog", "auth", "nginx", "k8s"]), + random.randint(10, 40))) + + attack_types = ["brute_force", "404_flood", "sqli", "port_scan"] + random.shuffle(attack_types) + for atk in attack_types[:random.randint(1, 3)]: + operations.append(("attack", atk, random.randint(15, 40))) + + random.shuffle(operations) + + for op_type, target, count in operations: + if op_type == "normal": + call_normal_script(target, count) + else: + call_attack_script(target, count) + + print(f"\nMix completado: {len(operations)} operaciones ejecutadas.\n") + + +# --------------------------------------------------------------------------- +# MODO 2: Solo ataques +# --------------------------------------------------------------------------- +def generate_attacks(): + """Funcion 2: llama solo a generate-test-logs.py con todos los ataques.""" + print("\n=== MODO ATAQUES: Solo logs de amenazas ===\n") + call_attack_script("all_attacks", 35) + print("\nAtaques completados.\n") + + +# --------------------------------------------------------------------------- +# MODO 3: Solo logs normales +# --------------------------------------------------------------------------- +def generate_normal(): + """Funcion 3: llama solo a generate-normal-logs.py con todas las fuentes.""" + print("\n=== MODO NORMAL: Solo logs legitimos ===\n") + call_normal_script("all", 30) + print("\nLogs normales completados.\n") + + +# --------------------------------------------------------------------------- +# MAIN +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description="Orquestador de logs SIEM (R8.5)") + parser.add_argument("--mode", choices=["random", "attacks", "normal"], + default="random", + help="Modo de generacion: random (mix), attacks (solo ataques), " + "normal (solo logs legitimos)") + args = parser.parse_args() + + print(f"Orquestador SIEM - {datetime.now(timezone.utc).isoformat()}") + + if args.mode == "random": + generate_random_mix() + elif args.mode == "attacks": + generate_attacks() + elif args.mode == "normal": + generate_normal() + + print("Esperar ~30s para que Filebeat procese los logs.") + print("Verificar en Kibana -> Discover -> indice logs-* o Security -> Alerts") + + +if __name__ == "__main__": + main() diff --git a/setup/provisioning/bootstrap-certs.sh b/setup/provisioning/bootstrap-certs.sh new file mode 100644 index 0000000..9385155 --- /dev/null +++ b/setup/provisioning/bootstrap-certs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env sh +set -eu +set -o pipefail 2>/dev/null || true + +CERT_DIR="config/certs" +INSTANCES_FILE="/setup/provisioning/cert-instances.yml" + +cd /usr/share/elasticsearch +mkdir -p "$CERT_DIR" + +if [ ! -f "$CERT_DIR/ca.zip" ]; then + echo "Generando CA..." + bin/elasticsearch-certutil ca --silent --pem -out "$CERT_DIR/ca.zip" + unzip "$CERT_DIR/ca.zip" -d "$CERT_DIR" +fi + +if [ ! -f "$CERT_DIR/certs.zip" ]; then + echo "Generando certificados de nodos..." + cp "$INSTANCES_FILE" "$CERT_DIR/instances.yml" + bin/elasticsearch-certutil cert --silent --pem \ + -out "$CERT_DIR/certs.zip" \ + --in "$CERT_DIR/instances.yml" \ + --ca-cert "$CERT_DIR/ca/ca.crt" \ + --ca-key "$CERT_DIR/ca/ca.key" + unzip "$CERT_DIR/certs.zip" -d "$CERT_DIR" +fi + +echo "Ajustando permisos de certificados..." +chown -R root:root "$CERT_DIR" +chmod -R 644 "$CERT_DIR" +find "$CERT_DIR" -type d -exec chmod 755 {} \; +echo "Setup completado." diff --git a/setup/provisioning/cert-instances.yml b/setup/provisioning/cert-instances.yml new file mode 100644 index 0000000..679465b --- /dev/null +++ b/setup/provisioning/cert-instances.yml @@ -0,0 +1,29 @@ +instances: + - name: es01 + dns: + - es01 + - localhost + ip: + - 127.0.0.1 + - name: es02 + dns: + - es02 + - localhost + ip: + - 127.0.0.1 + - name: es03 + dns: + - es03 + - localhost + ip: + - 127.0.0.1 + - name: kibana + dns: + - kibana + - localhost + ip: + - 127.0.0.1 + - name: logstash + dns: + - logstash + - localhost diff --git a/setup/provisioning/cleanup-indices.sh b/setup/provisioning/cleanup-indices.sh new file mode 100644 index 0000000..62dd6e5 --- /dev/null +++ b/setup/provisioning/cleanup-indices.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +set -e + +pip install -q requests urllib3 +python cleanup-indices.py --docker diff --git a/setup/provisioning/create-data-views.sh b/setup/provisioning/create-data-views.sh new file mode 100644 index 0000000..fa0326f --- /dev/null +++ b/setup/provisioning/create-data-views.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +set -e + +pip install -q requests urllib3 +python setup/create-data-views.py diff --git a/setup/provisioning/generate-logs.sh b/setup/provisioning/generate-logs.sh new file mode 100644 index 0000000..a042b08 --- /dev/null +++ b/setup/provisioning/generate-logs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +set -e + +sleep 15 +python setup/orchestrate-logs.py --mode normal diff --git a/setup/provisioning/ilm-setup.sh b/setup/provisioning/ilm-setup.sh new file mode 100644 index 0000000..077200d --- /dev/null +++ b/setup/provisioning/ilm-setup.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env sh +set -e + +echo "Esperando que el cluster este listo..." +sleep 15 + +echo "=== Configurando contrasena de kibana_system ===" +curl -s -X POST --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_security/user/kibana_system/_password" \ + -H "Content-Type: application/json" \ + -d "{\"password\":\"${KIBANA_PASSWORD}\"}" + +echo "" +echo "=== Creando ILM policy: siem-logs-policy (30 dias) ===" +curl -s -X PUT --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_ilm/policy/siem-logs-policy" \ + -H "Content-Type: application/json" \ + -d @/setup/ilm-policy.json + +if [ -f /setup/component-settings.json ]; then + echo "" + echo "=== Creando component templates ===" + curl -s -X PUT --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_component_template/siem-settings" \ + -H "Content-Type: application/json" \ + -d @/setup/component-settings.json +else + echo "=== Omitiendo component template: /setup/component-settings.json no existe ===" +fi + +echo "" +echo "=== Creando ingest pipelines ===" +if [ -f /setup/geoip-pipeline.json ]; then + curl -s -X PUT --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_ingest/pipeline/geoip-pipeline" \ + -H "Content-Type: application/json" \ + -d @/setup/geoip-pipeline.json + echo " Ingest pipeline: geoip-pipeline creado" +fi + +echo "" +echo "=== Creando index templates ===" +for tpl in siem-syslog siem-nginx siem-auth siem-k8s siem-errors logs-mapping; do + template_file="/setup/template-${tpl}.json" + if [ -f "$template_file" ]; then + curl -s -X PUT --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_index_template/${tpl}" \ + -H "Content-Type: application/json" \ + -d @"$template_file" + else + echo "Skipping missing index template: $template_file" + fi + echo "" +done + +echo "=== ILM y templates configurados correctamente ===" +curl -s --cacert /certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://es01:9200/_cluster/health?pretty" diff --git a/setup/provisioning/import-dashboards.sh b/setup/provisioning/import-dashboards.sh new file mode 100644 index 0000000..b7d986a --- /dev/null +++ b/setup/provisioning/import-dashboards.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +set -e + +sleep 10 +pip install -q requests urllib3 +python setup/create-dashboards-complete.py +python setup/import-dashboards.py diff --git a/setup/provisioning/import-rules.sh b/setup/provisioning/import-rules.sh new file mode 100644 index 0000000..3ed3a35 --- /dev/null +++ b/setup/provisioning/import-rules.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env sh +set -e + +apk add --no-cache curl > /dev/null + +echo "Actualizando all-rules.ndjson..." +python /setup/convert-rules-format.py + +echo "Esperando a que Kibana este disponible..." +until curl -s --cacert "${CACERT}" -o /dev/null -w "%{http_code}" "${KIBANA_HOST}" | grep -qE "^(200|302)$"; do + echo " esperando a Kibana..." + sleep 5 +done + +if [ "${SKIP_PREFLIGHT:-false}" = "true" ]; then + echo " SKIP_PREFLIGHT=true, omitiendo comprobacion de Security Solution..." +else + privileges_code=$(curl -s --cacert "${CACERT}" -o /tmp/detection-privileges.txt -w "%{http_code}" \ + -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" \ + -H "kbn-xsrf: true" \ + -H "Elastic-Api-Version: 2023-10-31" \ + "${KIBANA_HOST}/api/detection_engine/privileges") + if [ "${privileges_code}" != "200" ]; then + echo " Security Solution APIs are not available (HTTP ${privileges_code}); skipping rule import." + cat /tmp/detection-privileges.txt + exit 0 + fi +fi + +http_code=$(curl -s --cacert "${CACERT}" -o /tmp/response.txt -w "%{http_code}" \ + -X POST "${KIBANA_HOST}/api/detection_engine/rules/_import?overwrite=true" \ + -H "kbn-xsrf: true" \ + -H "Elastic-Api-Version: 2023-10-31" \ + -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" \ + --form "file=@/rules/all-rules.ndjson;type=application/x-ndjson") + +if [ "${http_code}" != "200" ] || ! grep -q '"success"[[:space:]]*:[[:space:]]*true' /tmp/response.txt; then + echo " Error importando reglas" + cat /tmp/response.txt + exit 1 +fi + +echo " Reglas importadas" diff --git a/setup/run-podman-sentinel-stack.sh b/setup/run-podman-sentinel-stack.sh new file mode 100755 index 0000000..6c52fa7 --- /dev/null +++ b/setup/run-podman-sentinel-stack.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV_FILE="${ENV_FILE:-$ROOT_DIR/.env}" +SENTINEL_ENV_FILE="${SENTINEL_ENV_FILE:-$ROOT_DIR/.env.sentinel}" +WITH_SENTINEL=true +BUILD=false +FORCE_RECREATE=false +PODMAN_COMPOSE_BIN="${PODMAN_COMPOSE:-}" + +usage() { + cat <<'EOF' +Usage: bash setup/run-podman-sentinel-stack.sh [--build] [--force-recreate] [--no-sentinel] [--compose-bin PATH] + +Starts CloudSIEM on rootless Podman in explicit stages. This avoids Podman +Compose failures with service_completed_successfully dependencies. + +Options: + --build Build images before starting long-running services. + --force-recreate Recreate long-running containers without deleting named volumes. + --no-sentinel Start only the local Elastic stack. + --compose-bin PATH Path to podman-compose binary. + -h, --help Show this help. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --build) + BUILD=true + ;; + --force-recreate) + FORCE_RECREATE=true + ;; + --no-sentinel) + WITH_SENTINEL=false + ;; + --compose-bin) + shift + PODMAN_COMPOSE_BIN="${1:-}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +find_podman_compose() { + if [ -n "$PODMAN_COMPOSE_BIN" ]; then + printf '%s\n' "$PODMAN_COMPOSE_BIN" + return + fi + if command -v podman-compose >/dev/null 2>&1; then + command -v podman-compose + return + fi + if [ -x "$ROOT_DIR/.venv-tools/podman-compose/bin/podman-compose" ]; then + printf '%s\n' "$ROOT_DIR/.venv-tools/podman-compose/bin/podman-compose" + return + fi + if [ -x "/tmp/opencode/podman-compose-venv/bin/podman-compose" ]; then + printf '%s\n' "/tmp/opencode/podman-compose-venv/bin/podman-compose" + return + fi + return 1 +} + +load_env_file() { + local file="$1" + local line="" + local key="" + local value="" + + if [ ! -f "$file" ]; then + echo "ERROR: required env file not found: $file" >&2 + exit 1 + fi + + while IFS= read -r line || [ -n "$line" ]; do + line="${line%$'\r'}" + if [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]]; then + continue + fi + line="${line#export }" + key="${line%%=*}" + value="${line#*=}" + + if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "ERROR: invalid env key in $file: $key" >&2 + exit 1 + fi + + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value="${value:1:${#value}-2}" + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value="${value:1:${#value}-2}" + fi + + export "$key=$value" + done < "$file" +} + +ensure_podman_socket() { + local socket_path="${XDG_RUNTIME_DIR}/podman/podman.sock" + if [ -S "$socket_path" ]; then + return 0 + fi + + mkdir -p "${XDG_RUNTIME_DIR}/podman" + + if [ -d "$socket_path" ]; then + echo "Found stale Podman socket directory: $socket_path" + if rmdir "$socket_path" 2>/dev/null; then + echo " Removed stale empty directory so podman.socket can bind." + else + cat >&2 </dev/null 2>&1; then + systemctl --user reset-failed podman.socket podman.service >/dev/null 2>&1 || true + systemctl --user start podman.socket >/dev/null 2>&1 || true + fi + + if [ ! -S "$socket_path" ]; then + cat >&2 </dev/null || true)" + if [ "$status" = "healthy" ] || { [ "$status" = "running" ] && [ "$container" = "siem-filebeat" ]; }; then + echo " $container is $status" + return 0 + fi + sleep 5 + done + + echo "ERROR: $container did not become healthy within ${timeout_seconds}s (last status: ${status:-unknown})" >&2 + podman logs --tail 120 "$container" >&2 || true + exit 1 +} + +wait_es_green() { + local timeout_seconds="$1" + local deadline=$((SECONDS + timeout_seconds)) + + echo "Waiting for Elasticsearch cluster green..." + while [ "$SECONDS" -lt "$deadline" ]; do + if podman exec siem-es01 curl -fsS \ + --cacert config/certs/ca/ca.crt \ + -u "elastic:${ELASTIC_PASSWORD}" \ + "https://localhost:9200/_cluster/health?wait_for_status=green&timeout=5s" \ + >/dev/null 2>&1; then + echo " Elasticsearch cluster is green" + return 0 + fi + sleep 5 + done + + echo "ERROR: Elasticsearch did not become green within ${timeout_seconds}s" >&2 + podman logs --tail 120 siem-es01 >&2 || true + exit 1 +} + +compose() { + "$PODMAN_COMPOSE_BIN" "${COMPOSE_FILES[@]}" "$@" +} + +run_job() { + local service="$1" + echo "Running one-shot service: $service" + compose run --rm --no-deps -T "$service" +} + +up_services() { + local fresh_services=() + local existing_services=() + local service="" + local args=() + + for service in "$@"; do + if podman inspect "$service" >/dev/null 2>&1; then + existing_services+=("$service") + else + fresh_services+=("$service") + fi + done + + if [ "${#fresh_services[@]}" -gt 0 ]; then + args=(up -d --no-deps) + if [ "$BUILD" = true ]; then + args+=(--build) + fi + echo "Starting services: ${fresh_services[*]}" + compose "${args[@]}" "${fresh_services[@]}" + fi + + if [ "${#existing_services[@]}" -gt 0 ]; then + args=(up -d --no-deps) + if [ "$BUILD" = true ]; then + args+=(--build) + fi + if [ "$FORCE_RECREATE" = true ]; then + args+=(--force-recreate) + fi + echo "Recreating services: ${existing_services[*]}" + compose "${args[@]}" "${existing_services[@]}" + fi +} + +cd "$ROOT_DIR" + +if ! command -v podman >/dev/null 2>&1; then + echo "ERROR: podman is required" >&2 + exit 1 +fi + +PODMAN_COMPOSE_BIN="$(find_podman_compose || true)" +if [ -z "$PODMAN_COMPOSE_BIN" ]; then + cat >&2 <<'EOF' +ERROR: podman-compose was not found. + +Install it locally with: + python3 -m venv .venv-tools/podman-compose + .venv-tools/podman-compose/bin/pip install podman-compose + +Then rerun this script. +EOF + exit 1 +fi + +if [ -z "${XDG_RUNTIME_DIR:-}" ]; then + echo "ERROR: XDG_RUNTIME_DIR must be set for rootless Podman" >&2 + exit 1 +fi +ensure_podman_socket + +load_env_file "$ENV_FILE" +if [ "$WITH_SENTINEL" = true ]; then + load_env_file "$SENTINEL_ENV_FILE" +fi + +: "${ELASTIC_PASSWORD:?ELASTIC_PASSWORD must be set in .env}" + +COMPOSE_FILES=(-f docker-compose.yml -f docker-compose-podman.yml) +if [ "$WITH_SENTINEL" = true ]; then + COMPOSE_FILES+=(-f docker-compose-sentinel.yml -f docker-compose-sentinel-podman.yml) +fi + +echo "Using podman-compose: $PODMAN_COMPOSE_BIN" +echo "Using compose files: ${COMPOSE_FILES[*]}" + +run_job setup +up_services es01 es02 es03 +wait_es_green 240 + +run_job cleanup-indices +run_job ilm-setup + +up_services kibana +wait_container_health siem-kibana 240 + +up_services logstash +wait_container_health siem-logstash 180 + +up_services filebeat +wait_container_health siem-filebeat 60 + +run_job rules-import +run_job data-views-setup +run_job dashboards-import +run_job log-generator + +if [ "$WITH_SENTINEL" = true ]; then + up_services sentinel-forwarder +fi + +echo "CloudSIEM Podman stack started successfully." diff --git a/setup/sentinel-forwarder.py b/setup/sentinel-forwarder.py new file mode 100644 index 0000000..07ceb8b --- /dev/null +++ b/setup/sentinel-forwarder.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +""" +sentinel-forwarder.py +Continuously forwards normalized CloudSIEM events from Elasticsearch to +Microsoft Sentinel through the Azure Monitor Logs Ingestion API. +""" + +import argparse +import base64 +import json +import os +import ssl +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +DEFAULT_INDEX = "logs-*" +TOKEN_SCOPE = "https://monitor.azure.com//.default" +INGEST_API_VERSION = "2023-01-01" +MAX_STRING_LENGTH = 16000 +MAX_INGEST_BYTES = 900000 + + +class ConfigError(RuntimeError): + """Raised when required runtime configuration is missing.""" + + +def getenv(name, default=None, required=False): + value = os.getenv(name, default) + if required and not value: + raise ConfigError(f"Missing required environment variable: {name}") + return value + + +def parse_args(): + parser = argparse.ArgumentParser(description="Forward CloudSIEM logs to Microsoft Sentinel") + parser.add_argument("--once", action="store_true", help="Run one polling iteration and exit") + parser.add_argument("--dry-run", action="store_true", help="Read and normalize events without sending to Azure") + return parser.parse_args() + + +def utc_now(): + return datetime.now(timezone.utc) + + +def iso_z(dt): + return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def parse_datetime(value): + if not value: + return None + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value / 1000, tz=timezone.utc) + text = str(value) + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def canonical_timestamp(value): + parsed = parse_datetime(value) + return iso_z(parsed or utc_now()) + + +def load_checkpoint(path): + checkpoint_path = Path(path) + if not checkpoint_path.exists(): + return {"timestamp": None, "ids": []} + try: + with checkpoint_path.open("r", encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, json.JSONDecodeError): + return {"timestamp": None, "ids": []} + return { + "timestamp": data.get("timestamp"), + "ids": data.get("ids", []), + } + + +def save_checkpoint(path, checkpoint): + checkpoint_path = Path(path) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = checkpoint_path.with_suffix(".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(checkpoint, handle, sort_keys=True) + tmp_path.replace(checkpoint_path) + + +def ssl_context(cacert=None, insecure=False): + if insecure: + return ssl._create_unverified_context() # noqa: S323 - explicit opt-in for local dev only + if cacert and Path(cacert).exists(): + return ssl.create_default_context(cafile=cacert) + return ssl.create_default_context() + + +def basic_auth_header(username, password): + token = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + return f"Basic {token}" + + +def request_json(method, url, headers=None, body=None, context=None, timeout=30): + data = None + request_headers = headers or {} + if body is not None: + data = json.dumps(body).encode("utf-8") + request_headers = {"Content-Type": "application/json", **request_headers} + + request = urllib.request.Request(url, data=data, headers=request_headers, method=method) + try: + with urllib.request.urlopen(request, timeout=timeout, context=context) as response: + response_body = response.read() + if not response_body: + return response.status, None + return response.status, json.loads(response_body.decode("utf-8")) + except urllib.error.HTTPError as exc: + response_body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {exc.code} for {url}: {response_body[:500]}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Request failed for {url}: {exc}") from exc + + +class AzureTokenProvider: + def __init__(self, tenant_id, client_id, client_secret): + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + self.access_token = None + self.expires_at = 0 + + def get_token(self): + if self.access_token and time.time() < self.expires_at - 120: + return self.access_token + + token_url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token" + payload = urllib.parse.urlencode( + { + "client_id": self.client_id, + "client_secret": self.client_secret, + "scope": TOKEN_SCOPE, + "grant_type": "client_credentials", + } + ).encode("utf-8") + request = urllib.request.Request( + token_url, + data=payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + token_payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + response_body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Azure token request failed: HTTP {exc.code}: {response_body[:500]}") from exc + + self.access_token = token_payload["access_token"] + self.expires_at = time.time() + int(token_payload.get("expires_in", 3600)) + return self.access_token + + +def get_path(document, path, default=None): + if path in document: + return document[path] + current = document + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return default + current = current[part] + return current + + +def first_value(value, default=None): + if isinstance(value, list): + for item in value: + if item not in (None, ""): + return item + return default + if value in (None, ""): + return default + return value + + +def as_string(value): + value = first_value(value, "") + if value is None: + return "" + if isinstance(value, (dict, list)): + text = json.dumps(value, ensure_ascii=False, sort_keys=True) + else: + text = str(value) + return text[:MAX_STRING_LENGTH] + + +def as_long(value): + value = first_value(value) + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def as_bool(value): + value = first_value(value) + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in ("true", "1", "yes") + return bool(value) if value is not None else False + + +def tags_value(document): + tags = get_path(document, "tags", []) + if tags is None: + return [] + if isinstance(tags, list): + return [str(tag) for tag in tags] + return [str(tags)] + + +def normalize_hit(hit): + document = hit.get("_source", {}) + timestamp = canonical_timestamp(get_path(document, "@timestamp")) + + return { + "TimeGenerated": timestamp, + "EventId": as_string(hit.get("_id")), + "ElasticIndex": as_string(hit.get("_index")), + "EventDataset": as_string(get_path(document, "event.dataset")), + "EventCategory": as_string(get_path(document, "event.category")), + "EventType": as_string(get_path(document, "event.type")), + "EventAction": as_string(get_path(document, "event.action")), + "EventOutcome": as_string(get_path(document, "event.outcome")), + "EventSeverity": as_long(get_path(document, "event.severity")), + "SiemSeverity": as_string(get_path(document, "siem.severity")), + "SourceIp": as_string(get_path(document, "source.ip")), + "SourcePort": as_long(get_path(document, "source.port")), + "DestinationIp": as_string(get_path(document, "destination.ip")), + "DestinationPort": as_long(get_path(document, "destination.port")), + "NetworkTransport": as_string(get_path(document, "network.transport")), + "UserName": as_string(get_path(document, "user.name")), + "HostName": as_string(get_path(document, "host.name")), + "ProcessName": as_string(get_path(document, "process.name")), + "HttpMethod": as_string(get_path(document, "http.request.method")), + "HttpStatusCode": as_long(get_path(document, "http.response.status_code")), + "HttpResponseBytes": as_long(get_path(document, "http.response.body.bytes")), + "UrlPath": as_string(get_path(document, "url.path")), + "UrlOriginal": as_string(get_path(document, "url.original")), + "UserAgent": as_string(get_path(document, "user_agent.original")), + "Tags": tags_value(document), + "BruteForceAttempt": as_bool(get_path(document, "siem.brute_force_attempt")), + "AfterHoursLogin": as_bool(get_path(document, "siem.after_hours_login")), + "SensitivePath": as_bool(get_path(document, "siem.sensitive_path")), + "SqliDetected": as_bool(get_path(document, "siem.sqli_detected")), + "XssDetected": as_bool(get_path(document, "siem.xss_detected")), + "PathTraversal": as_bool(get_path(document, "siem.path_traversal")), + "ScannerDetected": as_bool(get_path(document, "siem.scanner_detected")), + "Message": as_string(get_path(document, "message")), + } + + +def fetch_elasticsearch_batch(es_url, es_auth, es_context, index_pattern, checkpoint, batch_size, lookback_minutes): + last_ts = checkpoint.get("timestamp") + seen_ids = set(checkpoint.get("ids", [])) + start_time = last_ts or f"now-{lookback_minutes}m" + search_size = min(max(batch_size + len(seen_ids) + 100, batch_size), 10000) + search_body = { + "size": search_size, + "sort": [ + {"@timestamp": {"order": "asc", "unmapped_type": "date"}} + ], + "query": { + "range": { + "@timestamp": { + "gte": start_time + } + } + }, + } + endpoint = f"{es_url.rstrip('/')}/{index_pattern}/_search?ignore_unavailable=true&allow_no_indices=true" + _, payload = request_json( + "POST", + endpoint, + headers={"Authorization": es_auth}, + body=search_body, + context=es_context, + timeout=30, + ) + hits = payload.get("hits", {}).get("hits", []) if payload else [] + records = [] + event_ids = [] + for hit in hits: + record = normalize_hit(hit) + event_id = record["EventId"] + if last_ts and record["TimeGenerated"] == last_ts and event_id in seen_ids: + continue + records.append(record) + event_ids.append(event_id) + if len(records) >= batch_size: + break + return records, event_ids + + +def send_to_sentinel(token_provider, dce_endpoint, dcr_immutable_id, stream_name, records): + url = ( + f"{dce_endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}" + f"/streams/{stream_name}?api-version={INGEST_API_VERSION}" + ) + headers = { + "Authorization": f"Bearer {token_provider.get_token()}", + "Content-Type": "application/json", + } + data = json.dumps(records).encode("utf-8") + request = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=60) as response: + response.read() + if response.status < 200 or response.status >= 300: + raise RuntimeError(f"Unexpected Sentinel response status {response.status}") + except urllib.error.HTTPError as exc: + response_body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Sentinel ingestion failed: HTTP {exc.code}: {response_body[:500]}") from exc + + +def record_chunks(records, max_bytes=MAX_INGEST_BYTES): + chunk = [] + chunk_bytes = 2 + for record in records: + record_bytes = len(json.dumps(record).encode("utf-8")) + 1 + if chunk and chunk_bytes + record_bytes > max_bytes: + yield chunk + chunk = [] + chunk_bytes = 2 + chunk.append(record) + chunk_bytes += record_bytes + if chunk: + yield chunk + + +def update_checkpoint(previous, records, event_ids): + if not records: + return previous + + max_ts = max(record["TimeGenerated"] for record in records) + max_ids = [event_id for record, event_id in zip(records, event_ids) if record["TimeGenerated"] == max_ts] + if previous.get("timestamp") == max_ts: + max_ids.extend(previous.get("ids", [])) + return { + "timestamp": max_ts, + "ids": sorted(set(max_ids))[-10000:], + } + + +def main(): + args = parse_args() + try: + tenant_id = getenv("AZURE_TENANT_ID", required=not args.dry_run) + client_id = getenv("AZURE_CLIENT_ID", required=not args.dry_run) + client_secret = getenv("AZURE_CLIENT_SECRET", required=not args.dry_run) + dce_endpoint = getenv("SENTINEL_DCE_ENDPOINT", required=not args.dry_run) + dcr_immutable_id = getenv("SENTINEL_DCR_IMMUTABLE_ID", required=not args.dry_run) + except ConfigError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + stream_name = getenv("SENTINEL_STREAM_NAME", "Custom-CloudSIEM") + es_url = getenv("ES_URL", getenv("ELASTIC_HOSTS", "https://localhost:9200")) + es_url = es_url.split(",")[0] + es_user = getenv("ELASTIC_USER", "elastic") + es_password = getenv("ELASTIC_PASSWORD", required=True) + cacert = getenv("CACERT", "./setup/certs/ca/ca.crt") + checkpoint_file = getenv("SENTINEL_CHECKPOINT_FILE", "./sentinel-forwarder-checkpoint.json") + poll_seconds = int(getenv("SENTINEL_POLL_SECONDS", "10")) + batch_size = int(getenv("SENTINEL_BATCH_SIZE", "500")) + lookback_minutes = int(getenv("SENTINEL_LOOKBACK_MINUTES", "15")) + index_pattern = getenv("SENTINEL_INDEX_PATTERN", DEFAULT_INDEX) + insecure_es = getenv("SENTINEL_INSECURE_ES", "false").lower() == "true" + + es_context = ssl_context(cacert, insecure=insecure_es) + es_auth = basic_auth_header(es_user, es_password) + token_provider = None if args.dry_run else AzureTokenProvider(tenant_id, client_id, client_secret) + + print("CloudSIEM Sentinel forwarder started", flush=True) + print(f" Elasticsearch: {es_url}/{index_pattern}", flush=True) + print(f" Stream: {stream_name}", flush=True) + print(f" Checkpoint: {checkpoint_file}", flush=True) + + while True: + checkpoint = load_checkpoint(checkpoint_file) + try: + records, event_ids = fetch_elasticsearch_batch( + es_url, + es_auth, + es_context, + index_pattern, + checkpoint, + batch_size, + lookback_minutes, + ) + if records: + if args.dry_run: + print(f"Dry run: normalized {len(records)} records", flush=True) + else: + for chunk in record_chunks(records): + send_to_sentinel(token_provider, dce_endpoint, dcr_immutable_id, stream_name, chunk) + print(f"Forwarded {len(records)} records to Microsoft Sentinel", flush=True) + save_checkpoint(checkpoint_file, update_checkpoint(checkpoint, records, event_ids)) + else: + print("No new records to forward", flush=True) + except Exception as exc: # noqa: BLE001 - keep daemon alive after transient failures + print(f"ERROR: {exc}", file=sys.stderr, flush=True) + + if args.once: + break + time.sleep(poll_seconds) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup/template-filebeat.json b/setup/template-filebeat.json new file mode 100644 index 0000000..f34e09f --- /dev/null +++ b/setup/template-filebeat.json @@ -0,0 +1,12 @@ +{ + "index_patterns": ["filebeat", "filebeat-*"], + "data_stream": {}, + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.number_of_shards": 1, + "index.number_of_replicas": 0 + } + }, + "priority": 200 +} diff --git a/setup/template-logs-mapping.json b/setup/template-logs-mapping.json new file mode 100644 index 0000000..d1500a1 --- /dev/null +++ b/setup/template-logs-mapping.json @@ -0,0 +1,28 @@ +{ + "index_patterns": ["logs-*"], + "template": { + "mappings": { + "properties": { + "source": { + "properties": { + "geo": { + "properties": { + "location": { "type": "geo_point" } + } + } + } + }, + "destination": { + "properties": { + "geo": { + "properties": { + "location": { "type": "geo_point" } + } + } + } + } + } + } + }, + "priority": 500 +} diff --git a/setup/template-siem-auth.json b/setup/template-siem-auth.json new file mode 100644 index 0000000..5d6ac05 --- /dev/null +++ b/setup/template-siem-auth.json @@ -0,0 +1,41 @@ +{ + "index_patterns": ["logs-auth-*"], + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.lifecycle.rollover_alias": "logs-auth", + "index.number_of_shards": 1, + "index.number_of_replicas": 1 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + ], + "properties": { + "@timestamp": { "type": "date" }, + "source.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "source.port": { "type": "long" }, + "event.outcome": { "type": "keyword" }, + "event.action": { "type": "keyword" }, + "event.category": { "type": "keyword" }, + "user.name": { "type": "keyword" }, + "process.name": { "type": "keyword" }, + "host.name": { "type": "keyword" } + } + } + }, + "priority": 200 +} diff --git a/setup/template-siem-errors.json b/setup/template-siem-errors.json new file mode 100644 index 0000000..aca8ee5 --- /dev/null +++ b/setup/template-siem-errors.json @@ -0,0 +1,19 @@ +{ + "index_patterns": ["logs-errors-*"], + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.lifecycle.rollover_alias": "logs-errors", + "index.number_of_shards": 1, + "index.number_of_replicas": 1 + }, + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "error.type": { "type": "keyword" }, + "error.message": { "type": "text" } + } + } + }, + "priority": 200 +} diff --git a/setup/template-siem-k8s.json b/setup/template-siem-k8s.json new file mode 100644 index 0000000..b7b0336 --- /dev/null +++ b/setup/template-siem-k8s.json @@ -0,0 +1,19 @@ +{ + "index_patterns": ["logs-k8s-*"], + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.lifecycle.rollover_alias": "logs-k8s", + "index.number_of_shards": 1, + "index.number_of_replicas": 1 + }, + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "container.name": { "type": "keyword" }, + "kubernetes.pod.name": { "type": "keyword" } + } + } + }, + "priority": 200 +} diff --git a/setup/template-siem-nginx.json b/setup/template-siem-nginx.json new file mode 100644 index 0000000..93c959c --- /dev/null +++ b/setup/template-siem-nginx.json @@ -0,0 +1,65 @@ +{ + "index_patterns": ["logs-nginx-*"], + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.lifecycle.rollover_alias": "logs-nginx", + "index.number_of_shards": 1, + "index.number_of_replicas": 1 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + ], + "properties": { + "@timestamp": { "type": "date" }, + "client.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "source.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "destination.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "destination.port": { "type": "long" }, + "http.response.status_code": { "type": "long" }, + "event.dataset": { "type": "constant_keyword", "value": "nginx.access" }, + "url": { + "properties": { + "path": { + "type": "text", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 1024 } + } + }, + "original": { + "type": "text", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 2048 } + } + } + } + } + } + } + }, + "priority": 200 +} diff --git a/setup/template-siem-syslog.json b/setup/template-siem-syslog.json new file mode 100644 index 0000000..aa4a971 --- /dev/null +++ b/setup/template-siem-syslog.json @@ -0,0 +1,52 @@ +{ + "index_patterns": ["logs-syslog-*"], + "template": { + "settings": { + "index.lifecycle.name": "siem-logs-policy", + "index.lifecycle.rollover_alias": "logs-syslog", + "index.number_of_shards": 1, + "index.number_of_replicas": 1 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + ], + "properties": { + "@timestamp": { "type": "date" }, + "host.hostname": { "type": "keyword" }, + "host.name": { "type": "keyword" }, + "process.name": { "type": "keyword" }, + "log.level": { "type": "keyword" }, + "source.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "source.port": { "type": "long" }, + "destination.ip": { + "type": "keyword", + "fields": { + "ip": { "type": "ip" } + } + }, + "destination.port": { "type": "long" }, + "network.transport": { "type": "keyword" }, + "network.bytes": { "type": "long" }, + "network.ttl": { "type": "long" }, + "event.action": { "type": "keyword" }, + "event.category": { "type": "keyword" }, + "event.outcome": { "type": "keyword" } + } + } + }, + "priority": 200 +} diff --git a/setup/verify-cluster.sh b/setup/verify-cluster.sh new file mode 100755 index 0000000..ea57fcb --- /dev/null +++ b/setup/verify-cluster.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# ── Verificar estado del cluster ELK ───────────────────────────────────────── +set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../.env" + +ES_URL="https://localhost:9200" +AUTH="elastic:${ELASTIC_PASSWORD}" + +echo "╔══════════════════════════════════════════╗" +echo "║ Verificación del cluster SIEM ELK ║" +echo "╚══════════════════════════════════════════╝" +echo "" + +echo "── Estado del cluster ─────────────────────" +curl -s -k -u "$AUTH" "$ES_URL/_cluster/health?pretty" + +echo "" +echo "── Nodos activos ──────────────────────────" +curl -s -k -u "$AUTH" "$ES_URL/_cat/nodes?v" + +echo "" +echo "── ILM Policy ─────────────────────────────" +curl -s -k -u "$AUTH" "$ES_URL/_ilm/policy/siem-logs-policy?pretty" | head -30 + +echo "" +echo "── Índices existentes ──────────────────────" +curl -s -k -u "$AUTH" "$ES_URL/_cat/indices?v&s=index" + +echo "" +echo "── Fase ILM de cada índice ─────────────────" +curl -s -k -u "$AUTH" "$ES_URL/*/_ilm/explain?pretty" | (python3 -c " +import json, sys +data = json.load(sys.stdin) +for idx, info in data.get('indices', {}).items(): + phase = info.get('phase', 'N/A') + age = info.get('age', 'N/A') + print(f' {idx:40s} phase={phase:6s} age={age}') +" 2>/dev/null || cat) diff --git a/setup/verify-ecs-mapping.py b/setup/verify-ecs-mapping.py new file mode 100644 index 0000000..507e2ea --- /dev/null +++ b/setup/verify-ecs-mapping.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +""" +verify-ecs-mapping.py +Verifica que los campos ECS estén correctamente mapeados en Elasticsearch. +Comprueba las 4 fuentes de logs: syslog, nginx, auth, kubernetes. + +Uso: + pip install requests + python3 verify-ecs-mapping.py --host https://localhost:9200 \ + --user elastic --password SiemElastic2026! \ + --cacert ./setup/certs/ca/ca.crt +""" + +import argparse +import json +import sys +import urllib3 +import requests +from datetime import datetime, timezone, timedelta + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# ─── Campos ECS obligatorios por fuente ─────────────────────────────────────── +ECS_REQUIRED = { + "logs-syslog-*": [ + "@timestamp", "host.name", "message", + "event.dataset", "event.kind", "event.category", + "log.type", + ], + "logs-auth-*": [ + "@timestamp", "host.name", "message", + "event.dataset", "event.kind", "event.category", "event.outcome", + "event.action", "user.name", "source.ip", + ], + "logs-nginx-*": [ + "@timestamp", "source.ip", "message", + "http.request.method", "url.original", + "http.response.status_code", "http.response.body.bytes", + "event.dataset", "event.kind", "event.category", + ], + "logs-k8s-*": [ + "@timestamp", "message", + "kubernetes.pod.name", "kubernetes.namespace", + "kubernetes.container.name", "kubernetes.node.name", + "orchestrator.type", "orchestrator.namespace", + "event.dataset", "event.kind", + ], +} + +# ─── Colores para terminal ──────────────────────────────────────────────────── +GREEN = "\033[92m" +RED = "\033[91m" +YELLOW = "\033[93m" +BLUE = "\033[94m" +RESET = "\033[0m" +BOLD = "\033[1m" + + +def check(condition, ok_msg, fail_msg): + if condition: + print(f" {GREEN}✓{RESET} {ok_msg}") + return True + else: + print(f" {RED}✗{RESET} {fail_msg}") + return False + + +def get_index_stats(session, host, index_pattern, cacert): + """Obtiene estadísticas básicas del índice.""" + url = f"{host}/{index_pattern}/_stats/docs" + try: + r = session.get(url, verify=cacert, timeout=10) + if r.status_code == 404: + return None + r.raise_for_status() + data = r.json() + indices = data.get("indices", {}) + total_docs = sum( + v["primaries"]["docs"]["count"] + for v in indices.values() + ) + return {"doc_count": total_docs, "index_count": len(indices)} + except Exception as e: + return {"error": str(e)} + + +def get_sample_doc(session, host, index_pattern, cacert): + """Obtiene un documento de muestra del índice.""" + url = f"{host}/{index_pattern}/_search" + body = {"size": 1, "sort": [{"@timestamp": {"order": "desc"}}]} + try: + r = session.post(url, json=body, verify=cacert, timeout=10) + if r.status_code == 404: + return None + r.raise_for_status() + hits = r.json().get("hits", {}).get("hits", []) + return hits[0]["_source"] if hits else None + except Exception as e: + return None + + +def flatten_doc(doc, prefix=""): + """Aplana un documento anidado para verificar campos ECS.""" + result = {} + for k, v in doc.items(): + full_key = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + result.update(flatten_doc(v, full_key)) + else: + result[full_key] = v + return result + + +def verify_index(session, host, index_pattern, required_fields, cacert): + """Verifica un índice completo.""" + print(f"\n{BOLD}{BLUE}── {index_pattern}{RESET}") + + stats = get_index_stats(session, host, index_pattern, cacert) + if stats is None: + print(f" {YELLOW}⚠{RESET} Índice no encontrado (aún no hay logs)") + return {"status": "not_found", "pattern": index_pattern} + + if "error" in stats: + print(f" {RED}✗{RESET} Error: {stats['error']}") + return {"status": "error", "pattern": index_pattern} + + print(f" Documentos: {stats['doc_count']:,} en {stats['index_count']} índice(s)") + + if stats["doc_count"] == 0: + print(f" {YELLOW}⚠{RESET} Sin documentos — genera logs de prueba primero") + return {"status": "empty", "pattern": index_pattern} + + doc = get_sample_doc(session, host, index_pattern, cacert) + if not doc: + print(f" {YELLOW}⚠{RESET} No se pudo obtener muestra") + return {"status": "no_sample", "pattern": index_pattern} + + flat = flatten_doc(doc) + missing = [] + present = [] + + for field in required_fields: + if field in flat: + present.append(field) + else: + missing.append(field) + + # Mostrar resultado por campo + for f in required_fields: + val = flat.get(f, "MISSING") + if val != "MISSING": + display = str(val)[:60] + "..." if len(str(val)) > 60 else str(val) + check(True, f"{f} = {YELLOW}{display}{RESET}", "") + else: + check(False, "", f"{f} — campo ausente") + + coverage = len(present) / len(required_fields) * 100 + color = GREEN if coverage == 100 else (YELLOW if coverage >= 80 else RED) + print(f"\n Cobertura ECS: {color}{coverage:.0f}%{RESET} " + f"({len(present)}/{len(required_fields)} campos)") + + return { + "status": "ok" if not missing else "partial", + "pattern": index_pattern, + "coverage": coverage, + "missing": missing, + "doc_count": stats["doc_count"], + } + + +def check_cluster_health(session, host, cacert): + """Verifica el estado del cluster.""" + print(f"\n{BOLD}=== Estado del cluster ==={RESET}") + try: + r = session.get(f"{host}/_cluster/health", verify=cacert, timeout=10) + r.raise_for_status() + h = r.json() + color = h.get("status", "unknown") + emoji = {"green": "🟢", "yellow": "🟡", "red": "🔴"}.get(color, "⚪") + print(f" Estado: {emoji} {color.upper()}") + print(f" Nodos: {h.get('number_of_nodes', 0)} total, " + f"{h.get('number_of_data_nodes', 0)} data") + print(f" Shards: {h.get('active_shards', 0)} activos, " + f"{h.get('unassigned_shards', 0)} sin asignar") + return color == "green" + except Exception as e: + print(f" {RED}Error conectando al cluster: {e}{RESET}") + return False + + +def check_ilm_policy(session, host, cacert): + """Verifica que el ILM policy esté activo.""" + print(f"\n{BOLD}=== ILM Policy: siem-logs-policy ==={RESET}") + try: + r = session.get( + f"{host}/_ilm/policy/siem-logs-policy", + verify=cacert, timeout=10 + ) + if r.status_code == 404: + check(False, "", "ILM policy 'siem-logs-policy' no encontrada — ejecutar init-ilm.sh") + return False + r.raise_for_status() + policy = r.json().get("siem-logs-policy", {}).get("policy", {}) + phases = policy.get("phases", {}) + for phase in ["hot", "warm", "cold", "delete"]: + check(phase in phases, f"Fase '{phase}' configurada", f"Fase '{phase}' ausente") + delete_age = ( + phases.get("delete", {}) + .get("actions", {}) + .get("delete", {}) + ) + age = phases.get("delete", {}).get("min_age", "?") + check(True, f"Retención configurada a {age}", "") + return True + except Exception as e: + print(f" {RED}Error: {e}{RESET}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="Verificar mapeo ECS en índices SIEM") + parser.add_argument("--host", default="https://localhost:9200") + parser.add_argument("--user", default="elastic") + parser.add_argument("--password", default="SiemElastic2026!") + parser.add_argument("--cacert", default="./setup/certs/ca/ca.crt") + parser.add_argument("--insecure", action="store_true", + help="Deshabilitar verificación de certificado") + args = parser.parse_args() + + print(f"\n{BOLD}{'='*50}") + print(" Verificación ECS — SIEM Elastic Stack") + print(f" {datetime.now(timezone(timedelta(hours=-5))).strftime('%Y-%m-%d %H:%M:%S UTC-5')}") + print(f"{'='*50}{RESET}") + print(f" Host: {args.host}") + print(f" Usuario: {args.user}") + + session = requests.Session() + session.auth = (args.user, args.password) + cacert = False if args.insecure else args.cacert + + # 1. Cluster health + cluster_ok = check_cluster_health(session, args.host, cacert) + + # 2. ILM policy + ilm_ok = check_ilm_policy(session, args.host, cacert) + + # 3. Verificar cada fuente de logs + print(f"\n{BOLD}=== Verificación de campos ECS por fuente ==={RESET}") + results = [] + for pattern, fields in ECS_REQUIRED.items(): + result = verify_index(session, args.host, pattern, fields, cacert) + results.append(result) + + # 4. Resumen + print(f"\n{BOLD}{'='*50}") + print(" Resumen") + print(f"{'='*50}{RESET}") + print(f" Cluster: {'🟢 green' if cluster_ok else '🔴 problema'}") + print(f" ILM: {'✓ ok' if ilm_ok else '✗ no configurado'}") + print() + + all_ok = True + coverages = [] + for r in results: + status = r.get("status", "unknown") + pattern = r.get("pattern", "") + coverage = r.get("coverage", 0) + coverages.append(coverage) + + if status == "ok": + print(f" {GREEN}✓{RESET} {pattern:30s} 100% ECS") + elif status == "partial": + missing = ", ".join(r.get("missing", [])) + print(f" {YELLOW}~{RESET} {pattern:30s} {coverage:.0f}% — falta: {missing}") + all_ok = False + elif status in ("not_found", "empty"): + print(f" {YELLOW}⚠{RESET} {pattern:30s} sin datos aún") + else: + print(f" {RED}✗{RESET} {pattern:30s} error") + all_ok = False + + avg = sum(coverages) / len(coverages) if coverages else 0 + print(f"\n Cobertura promedio: {avg:.0f}%") + + # Exit code para CI/CD + sys.exit(0 if all_ok else 1) + + +if __name__ == "__main__": + main() diff --git a/setup/verify-parse-rate.py b/setup/verify-parse-rate.py new file mode 100644 index 0000000..e72e06c --- /dev/null +++ b/setup/verify-parse-rate.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +verify-parse-rate.py +Verifica que la tasa de parseo del pipeline Logstash sea superior al 95% (R8.3). +Conecta a Elasticsearch, cuenta documentos en indices exitosos vs errores. + +Uso: + python3 verify-parse-rate.py + python3 verify-parse-rate.py --host https://localhost:9200 --insecure + python3 verify-parse-rate.py --host https://localhost:9200 --user elastic --password +""" + +import argparse +import os +import sys +import requests +from datetime import datetime, timezone, timedelta +from pathlib import Path + +urllib3_disable_warnings = False +try: + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +except ImportError: + pass + +# ─── Configuración ──────────────────────────────────────────────────────────────── +THRESHOLD = 95.0 +EXIT_INDICES = [ + "logs-syslog-*", + "logs-auth-*", + "logs-nginx-*", + "logs-k8s-*", +] +ERROR_INDEX = "logs-errors-*" + +# Colores +G = "\033[92m"; R = "\033[91m"; Y = "\033[93m" +B = "\033[94m"; W = "\033[1m"; X = "\033[0m"; CYAN = "\033[36m" +BOLD = "\033[1m"; RESET = "\033[0m" + +def count_docs(session, host, index_pattern, cacert): + """Cuenta documentos en un patrón de índices.""" + url = f"{host}/{index_pattern}/_count" + try: + r = session.get(url, verify=cacert, timeout=10) + if r.status_code == 404: + return 0 + r.raise_for_status() + return r.json().get("count", 0) + except Exception as e: + raise RuntimeError(f"Error consultando {index_pattern}: {e}") + +def resolve_cacert(cacert_path, insecure): + """Resuelve el CA para requests o desactiva TLS si no existe.""" + if insecure: + return False + + if cacert_path and Path(cacert_path).is_file(): + return cacert_path + + print( + f"{Y} Advertencia:{RESET} no existe el CA en {cacert_path}; " + "se desactiva la verificación TLS.", + file=sys.stderr, + ) + return False + +def main(): + parser = argparse.ArgumentParser(description="Verificar tasa de parseo de logs") + parser.add_argument("--host", default="https://localhost:9200") + parser.add_argument("--user", default="elastic") + parser.add_argument("--password", default="SiemElastic2026!") + parser.add_argument("--cacert", default=os.getenv("CACERT", "./setup/certs/ca/ca.crt")) + parser.add_argument("--insecure", action="store_true", help="Deshabilitar verificación TLS") + args = parser.parse_args() + + print(f"{BOLD}{'='*55}{RESET}") + print(f" Verificacion de Tasa de Parseo - R8.3") + print(f" {datetime.now(timezone(timedelta(hours=-5))).strftime('%Y-%m-%d %H:%M:%S UTC-5')}") + print(f"{BOLD}{'='*55}{RESET}") + print(f" Host Elasticsearch: {args.host}") + print(f" Umbral requerido: > {THRESHOLD}%\n") + + session = requests.Session() + session.auth = (args.user, args.password) + cacert = resolve_cacert(args.cacert, args.insecure) + + total_exit = 0 + counts_by_index = {} + + print(f"{CYAN}--- Conteo de documentos por indice ---{RESET}") + + try: + for pattern in EXIT_INDICES: + count = count_docs(session, args.host, pattern, cacert) + counts_by_index[pattern] = count + total_exit += count + status = f"{G}{count:>8,}{RESET}" + print(f" {pattern:20s} {status} docs (parseo exitoso)") + + errors = count_docs(session, args.host, ERROR_INDEX, cacert) + except RuntimeError as exc: + print(f"\n{R} Error verificando Elasticsearch:{RESET} {exc}") + sys.exit(2) + + counts_by_index[ERROR_INDEX] = errors + status = f"{R if errors > 0 else G}{errors:>8,}{RESET}" + print(f" {'logs-errors-*':20s} {status} docs (errores de parseo)") + + total_all = total_exit + errors + + if total_all == 0: + print(f"\n{Y} No hay datos en Elasticsearch.{RESET}") + print(" Genera logs primero: python3 setup/generate-normal-logs.py --source all") + sys.exit(0) + + parse_rate = (total_exit / total_all) * 100 + + print(f"\n{CYAN}--- Resultado ---{RESET}") + print(f" Total documentos procesados: {total_all:>8,}") + print(f" Parseo exitoso: {total_exit:>8,}") + print(f" Parseo fallido (errores): {errors:>8,}") + print(f"") + + if parse_rate >= THRESHOLD: + print(f" Tasa de parseo: {G}{parse_rate:.2f}%{RESET} >= {THRESHOLD}% -> {G}OK{RESET}") + else: + print(f" Tasa de parseo: {R}{parse_rate:.2f}%{RESET} < {THRESHOLD}% -> {R}FALLIDO{RESET}") + print(f"\n {Y}Revision recomendada:{RESET}") + print(f" - Verificar patrones grok en logstash/pipeline/") + print(f" - Revisar logs de Logstash: docker logs siem-logstash | tail -50") + print(f" - Verificar indice de errores: curl -k {args.host}/{ERROR_INDEX}/_search?pretty") + + print(f"\n{CYAN}--- Detalle por fuente ---{RESET}") + for pattern, count in counts_by_index.items(): + pct = (count / total_all * 100) if total_all > 0 else 0 + bar_len = 40 + filled = int(bar_len * pct / 100) + bar = f"{'#' * filled}{'-' * (bar_len - filled)}" + print(f" {pattern:20s} [{bar}] {pct:5.1f}%") + + passed = parse_rate >= THRESHOLD + print(f"\n{BOLD}{'='*55}{RESET}") + if passed: + print(f" {G}CONCLUSION: R8.3 CUMPLE (tasa {parse_rate:.2f}% > {THRESHOLD}%){RESET}") + else: + print(f" {R}CONCLUSION: R8.3 NO CUMPLE (tasa {parse_rate:.2f}% < {THRESHOLD}%){RESET}") + print(f"{BOLD}{'='*55}{RESET}\n") + + sys.exit(0 if passed else 1) + +if __name__ == "__main__": + main() diff --git a/setup/verify-sentinel.py b/setup/verify-sentinel.py new file mode 100644 index 0000000..7c390ec --- /dev/null +++ b/setup/verify-sentinel.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +verify-sentinel.py +Verifies Microsoft Sentinel resources and recent CloudSIEM ingestion through Azure CLI. +""" + +import argparse +import json +import subprocess +import sys + + +def run_az(args, text_output=False): + command = ["az", *args] + result = subprocess.run(command, check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Command failed: {' '.join(command)}\n{result.stderr.strip()}") + output = result.stdout.strip() + if text_output: + return output + if not output: + return None + return json.loads(output) + + +def first_result_cell(query_payload): + if isinstance(query_payload, list): + if not query_payload: + return None + first_row = query_payload[0] + if not isinstance(first_row, dict): + return first_row + for key in ("Count", "count_", "count"): + if key in first_row: + return first_row[key] + for key, value in first_row.items(): + if key not in ("TableName", "$table"): + return value + return None + + tables = query_payload.get("tables", []) if isinstance(query_payload, dict) else [] + if not tables or not tables[0].get("rows"): + return None + first_row = tables[0]["rows"][0] + return first_row[0] if first_row else None + + +def query_workspace(workspace_id, query): + return run_az( + [ + "monitor", + "log-analytics", + "query", + "--workspace", + workspace_id, + "--analytics-query", + query, + "--output", + "json", + ] + ) + + +def main(): + parser = argparse.ArgumentParser(description="Verify CloudSIEM Microsoft Sentinel integration") + parser.add_argument("--resource-group", default="rg-cloudsiem-sentinel") + parser.add_argument("--workspace", default="law-cloudsiem") + parser.add_argument("--table", default="CloudSIEM_CL") + parser.add_argument("--subscription", default=None) + args = parser.parse_args() + + try: + if args.subscription: + run_az(["account", "set", "--subscription", args.subscription], text_output=True) + + subscription_id = run_az(["account", "show", "--query", "id", "--output", "tsv"], text_output=True) + workspace_id = run_az( + [ + "monitor", + "log-analytics", + "workspace", + "show", + "--resource-group", + args.resource_group, + "--workspace-name", + args.workspace, + "--query", + "customerId", + "--output", + "tsv", + ], + text_output=True, + ) + + count_payload = query_workspace(workspace_id, f"{args.table} | summarize Count=count()") + total_count = first_result_cell(count_payload) + + recent_payload = query_workspace( + workspace_id, + f"{args.table} | where TimeGenerated > ago(30m) | summarize Count=count()", + ) + recent_count = first_result_cell(recent_payload) + + rules_uri = ( + f"https://management.azure.com/subscriptions/{subscription_id}" + f"/resourceGroups/{args.resource_group}" + f"/providers/Microsoft.OperationalInsights/workspaces/{args.workspace}" + "/providers/Microsoft.SecurityInsights/alertRules" + "?api-version=2023-02-01-preview" + ) + rules_payload = run_az(["rest", "--method", "get", "--uri", rules_uri, "--output", "json"]) + cloudsiem_rules = [ + rule for rule in rules_payload.get("value", []) + if rule.get("properties", {}).get("displayName", "").startswith("CloudSIEM -") + ] + + print("Microsoft Sentinel verification") + print(f" Subscription: {subscription_id}") + print(f" Workspace: {args.workspace} ({workspace_id})") + print(f" Table: {args.table}") + print(f" Total rows: {total_count}") + print(f" Last 30 min: {recent_count}") + print(f" Rules: {len(cloudsiem_rules)} CloudSIEM rules") + + if total_count is None: + print("ERROR: no query result returned for the CloudSIEM table", file=sys.stderr) + return 1 + if len(cloudsiem_rules) < 5: + print("ERROR: expected at least 5 CloudSIEM analytics rules", file=sys.stderr) + return 1 + except Exception as exc: # noqa: BLE001 - CLI diagnostics should be shown as-is + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main())