Development

.gitignore

@@ -195,3 +195,8 @@ cython_debug/
 
 # Logs folder
 logs/
+
+# Container folder
+container/run/
+container/logs/
+container/fluentlogs/

demo/Dockerfile → Dockerfile

@@ -13,7 +13,6 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 COPY pyproject.toml README.md ./
 COPY ./src ./src
-COPY ./demo ./demo
 COPY ./tests ./tests
 
 RUN uv pip install --system -e .

container/Dockerfile_fluentd

@@ -0,0 +1,10 @@
+FROM fluent/fluentd:latest
+
+USER root
+
+RUN apt update -y && apt install -y make build-essential
+
+RUN fluent-gem install fluent-plugin-parser-protobuf
+RUN fluent-gem install fluent-plugin-detectmate
+RUN fluent-gem install fluent-plugin-nng
+RUN fluent-gem install fluent-plugin-kafka

container/config/detector_config.yaml

@@ -0,0 +1,9 @@
+detectors:
+  NewValueDetector:
+   method_type: new_value_detector
+   data_use_training: 2
+   auto_config: false
+   global:  # define global instance for new_value_detector similar to "events"
+     global_instance:  # define instance name
+       header_variables:  # another level to have the same structure as "events"
+         - pos: URL

container/config/detector_settings.yaml

@@ -0,0 +1,17 @@
+component_name: NewValueDetector01
+component_type: NewValueDetector  # this is the component class name, the service will automatically search for the matching component config schema
+log_level: "DEBUG"
+log_dir: "/logs"
+
+# Manager Interface (Command Channel)
+http_host: 0.0.0.0
+
+# Engine Interface (Data Channel)
+engine_addr: "ipc:///run/detector.ipc"
+engine_autostart: true
+
+# Output Destinations (where processed data is sent)
+out_addr:
+  - "ipc:///run/output.ipc"
+
+out_dial_timeout: 1000

container/config/parser_config.yaml

@@ -0,0 +1,11 @@
+parsers:
+  MatcherParser:
+    method_type: matcher_parser
+    auto_config: false
+    log_format: '<IP> - - [<Time>] "<Method> <URL> <Protocol>" <Status> <Bytes> "<Referer>" "<UserAgent>"'
+    time_format: null
+    params:
+      remove_spaces: false
+      remove_punctuation: false
+      lowercase: false
+      path_templates: /config/templates.txt  # empty file because there are no templates necessary for apache access logs

container/config/parser_settings.yaml

@@ -0,0 +1,17 @@
+component_name: accesslogparser
+component_type: MatcherParser
+log_level: "DEBUG"
+log_dir: "/logs"
+
+# Manager Interface (Command Channel)
+http_host: 0.0.0.0
+
+# Engine Interface (Data Channel)
+engine_addr: "ipc:///run/parser.engine.ipc"
+engine_autostart: true
+
+# Output Destinations (where processed data is sent)
+out_addr:
+  - "ipc:///run/detector.ipc"
+
+out_dial_timeout: 1000

container/fluentin/fluent.conf

@@ -0,0 +1,168 @@
+####
+## Output descriptions:
+##
+
+
+# Treasure Data (http://www.treasure-data.com/) provides cloud based data
+# analytics platform, which easily stores and processes data from td-agent.
+# FREE plan is also provided.
+# @see http://docs.fluentd.org/articles/http-to-td
+#
+# This section matches events whose tag is td.DATABASE.TABLE
+# <match td.*.*>
+#   @type tdlog
+#   @id output_td
+#   apikey YOUR_API_KEY
+#
+#   auto_create_table
+#   <buffer>
+#     @type file
+#     path /var/log/fluent/buffer/td
+#   </buffer>
+#
+#   <secondary>
+#     @type secondary_file
+#     directory /var/log/fluent/failed_records
+#   </secondary>
+# </match>
+
+## match tag=debug.** and dump to console
+# <match debug.**>
+#   @type stdout
+#   @id output_stdout
+# </match>
+
+####
+## Source descriptions:
+##
+
+## built-in TCP input
+## @see http://docs.fluentd.org/articles/in_forward
+# <source>
+#   @type forward
+#   @id input_forward
+# </source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# POST http://localhost:8888/<tag>?json=<json>
+# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
+# @see http://docs.fluentd.org/articles/in_http
+# <source>
+#   @type http
+#   @id input_http
+#   port 8888
+# </source>
+
+## live debugging agent
+# <source>
+#   @type debug_agent
+#   @id input_debug_agent
+#   bind 127.0.0.1
+#   port 24230
+# </source>
+
+####
+## Examples:
+##
+#
+## File input
+## read apache logs continuously and tags td.apache.access
+#<source>
+#  @type tail
+#  @id input_tail
+#  <parse>
+#    @type apache2
+#  </parse>
+#  path /var/log/httpd-access.log
+#  tag td.apache.access
+#</source>
+
+## File output
+## match tag=local.** and write to file
+#<match local.**>
+#  @type file
+#  @id output_file
+#  path /var/log/fluent/access
+#</match>
+
+## Forwarding
+## match tag=system.** and forward to another fluentd server
+#<match system.**>
+#  @type forward
+#  @id output_system_forward
+#
+#  <server>
+#    host 192.168.0.11
+#  </server>
+#  # secondary host is optional
+#  <secondary>
+#    <server>
+#      host 192.168.0.12
+#    </server>
+#  </secondary>
+#</match>
+
+## Multiple output
+## match tag=td.*.* and output to Treasure Data AND file
+#<match td.*.*>
+#  @type copy
+#  @id output_copy
+#  <store>
+#    @type tdlog
+#    apikey API_KEY
+#    auto_create_table
+#    <buffer>
+#      @type file
+#      path /var/log/fluent/buffer/td
+#    </buffer>
+#  </store>
+#  <store>
+#    @type file
+#    path /var/log/fluent/td-%Y-%m-%d/%H.log
+#  </store>
+#</match>
+#
+# <source>
+#   @type tail
+#   @id input_tail
+#   <parse>
+#   @type none
+# #    @type syslog
+# #    time_format "%Y-%m-%dT%H:%M:%S.%N%z"
+#   </parse>
+#   path /var/log/syslog
+#   tag syslog.*
+# </source>
+
+<source>
+  @type tail
+  @id input_tail
+  <parse>
+    @type none
+  </parse>
+  path /fluentd/log/some.log
+  path_key logSource
+
+  tag nng.*
+</source>
+
+<match nng.**>
+  @type nng
+  uri ipc:///run/parser.engine.ipc
+  <inject>
+    hostname_key hostname
+    # overwrite hostname:
+    # hostname somehost
+  </inject>
+  <buffer>
+    flush_mode immediate
+  </buffer>
+  <format>
+    @type detectmate
+  </format>
+</match>

container/fluentout/fluent.conf

@@ -0,0 +1,55 @@
+
+
+
+<source>
+  @type nng
+  @id input_nng
+  <parse>
+#     @type none
+#    @type detectmate
+    @type protobuf
+    class_file /fluentd/etc/schemas_pb.rb
+    class_name DetectorSchema
+    protobuf_version protobuf3
+  </parse>
+  uri ipc:///run/output.ipc
+  tag nng.*
+</source>
+
+<match nng.**>
+  @type file
+  path /fluentd/log/output.%Y%m%d
+  compress text
+  <buffer>
+    flush_mode immediate
+    timekey 1d
+    timekey_use_utc true
+  </buffer>
+</match>
+
+# <match nng.**>
+#   @type stdout
+# </match>
+
+
+# EXAMPLE KAFKA OUTPUT:
+# <match **>
+#   @type kafka2
+#
+#   # list of seed brokers
+#   brokers kafka-broker-0:9092,kafka-broker-1:9092,kafka-broker-2:9092
+# #  use_event_time true
+#
+#   # data type settings
+#   <format>
+#     @type json
+#   </format>
+#
+#   # topic settings
+#   topic_key detectmate
+#   default_topic detectmate
+#
+#   # producer settings
+#  # required_acks -1
+#   @log_level debug
+# </match>

container/grafana/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

container/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s  # Frequency of metric scraping
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['parser:8000', 'detector:8000']