From d1e2ba41e6c29d12af72e1459692662046d55024 Mon Sep 17 00:00:00 2001 From: whotwagner Date: Mon, 16 Mar 2026 16:17:31 +0000 Subject: [PATCH 01/22] Added docker-compose with fluentd pipeline --- .gitignore | 5 + Dockerfile | 21 +++ container/Dockerfile_fluentd | 10 ++ container/config/detector_config.yaml | 9 ++ container/config/detector_settings.yaml | 19 +++ container/config/parser_config.yaml | 11 ++ container/config/parser_settings.yaml | 19 +++ container/config/templates.txt | 0 container/fluentin/fluent.conf | 168 ++++++++++++++++++++++++ container/fluentout/fluent.conf | 161 +++++++++++++++++++++++ container/fluentout/schemas_pb.rb | 17 +++ container/grafana/prometheus.yml | 9 ++ container/prometheus.yml | 7 + docker-compose.yml | 106 +++++++++++++++ 14 files changed, 562 insertions(+) create mode 100644 Dockerfile create mode 100644 container/Dockerfile_fluentd create mode 100644 container/config/detector_config.yaml create mode 100644 container/config/detector_settings.yaml create mode 100644 container/config/parser_config.yaml create mode 100644 container/config/parser_settings.yaml create mode 100644 container/config/templates.txt create mode 100644 container/fluentin/fluent.conf create mode 100644 container/fluentout/fluent.conf create mode 100644 container/fluentout/schemas_pb.rb create mode 100644 container/grafana/prometheus.yml create mode 100644 container/prometheus.yml create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index dfea432b..cac2d6f1 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,8 @@ cython_debug/ # Logs folder logs/ + +# Container folder +container/run/ +container/logs/ +container/fluentlogs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e63255e4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +#RUN pip install uv + +COPY pyproject.toml README.md ./ +COPY ./src ./src +COPY ./demo ./demo +COPY ./tests ./tests + +RUN uv pip install --system -e . + +CMD ["detectmate", "--help"] diff --git a/container/Dockerfile_fluentd b/container/Dockerfile_fluentd new file mode 100644 index 00000000..e57239db --- /dev/null +++ b/container/Dockerfile_fluentd @@ -0,0 +1,10 @@ +FROM fluent/fluentd:latest + +USER root + +RUN apt update -y && apt install -y make build-essential + +RUN fluent-gem install fluent-plugin-parser-protobuf +RUN fluent-gem install fluent-plugin-detectmate +RUN fluent-gem install fluent-plugin-nng +RUN fluent-gem install fluent-plugin-kafka diff --git a/container/config/detector_config.yaml b/container/config/detector_config.yaml new file mode 100644 index 00000000..22b0a600 --- /dev/null +++ b/container/config/detector_config.yaml @@ -0,0 +1,9 @@ +detectors: + NewValueDetector: + method_type: new_value_detector + data_use_training: 2 + auto_config: false + global: # define global instance for new_value_detector similar to "events" + global_instance: # define instance name + header_variables: # another level to have the same structure as "events" + - pos: Status diff --git a/container/config/detector_settings.yaml b/container/config/detector_settings.yaml new file mode 100644 index 00000000..3e516fa7 --- /dev/null +++ b/container/config/detector_settings.yaml @@ -0,0 +1,19 @@ +component_name: NewValueDetector01 +component_type: NewValueDetector +log_level: "DEBUG" +log_dir: "/logs" + +http_host: 0.0.0.0 + +# Manager Interface (Command Channel) +manager_addr: "ipc:///run/detector.cmd.ipc" + +# Engine Interface (Data Channel) +engine_addr: "ipc:///run/detector.ipc" +engine_autostart: true + +# Output Destinations (where processed data is sent) +out_addr: + - "ipc:///run/output.ipc" + +out_dial_timeout: 1000 diff --git a/container/config/parser_config.yaml b/container/config/parser_config.yaml new file mode 100644 index 00000000..1b865ea9 --- /dev/null +++ b/container/config/parser_config.yaml @@ -0,0 +1,11 @@ +parsers: + MatcherParser: + method_type: matcher_parser + auto_config: false + log_format: ' - - [