diff --git a/pipeline/filters/checklist.md b/pipeline/filters/checklist.md index 86caa24f1..5a25d578d 100644 --- a/pipeline/filters/checklist.md +++ b/pipeline/filters/checklist.md @@ -48,23 +48,23 @@ pipeline: ```text [INPUT] - name tail - tag test1 - path test1.log - read_from_head true - parser json + Name tail + Tag test1 + Path test1.log + Read_From_Head true + Parser json [FILTER] - name checklist - match test1 - file ip_list.txt - lookup_key $remote_addr - record ioc abc - record badurl null + Name checklist + Match test1 + File ip_list.txt + Lookup_Key $remote_addr + Record ioc abc + Record badurl null [OUTPUT] - name stdout - match test1 + Name stdout + Match test1 ``` {% endtab %} diff --git a/pipeline/filters/ecs-metadata.md b/pipeline/filters/ecs-metadata.md index 2899d9566..b608e5ab1 100644 --- a/pipeline/filters/ecs-metadata.md +++ b/pipeline/filters/ecs-metadata.md @@ -25,16 +25,16 @@ The following template variables can be used for values with the `add` option. S | Variable | Description | Supported with `cluster_metadata_only` on | | :--- | :--- | :--- | | `$ClusterName` | The ECS cluster name. Fluent Bit is running on EC2 instances that are part of this cluster. | `Yes` | +| `$ContainerID` | The ID of the container from which the log originated. This is the full 64-character-long container ID. | `No` | | `$ContainerInstanceArn` | The full ARN of the ECS EC2 Container Instance. This is the instance that Fluent Bit is running on. | `Yes` | | `$ContainerInstanceID` | The ID of the ECS EC2 Container Instance. | `Yes` | +| `$DockerContainerName` | The name of the container from which the log originated. This is the name obtained from Docker and is the name shown if you run `docker ps` on the instance. | `No` | | `$ECSAgentVersion` | The version string of the ECS Agent running on the container instance. | `Yes` | | `$ECSContainerName` | The name of the container from which the log originated. This is the name in your ECS Task Definition. | `No` | -| `$DockerContainerName` | The name of the container from which the log originated. This is the name obtained from Docker and is the name shown if you run `docker ps` on the instance. | `No` | -| `$ContainerID` | The ID of the container from which the log originated. This is the full 64-character-long container ID. | `No` | +| `$TaskARN` | The full ARN of the ECS Task from which the log originated. | `No` | | `$TaskDefinitionFamily` | The family name of the task definition for the task from which the log originated. | `No` | | `$TaskDefinitionVersion` | The version or revision of the task definition for the task from which the log originated. | `No` | | `$TaskID` | The ID of the ECS Task from which the log originated. | `No` | -| `$TaskARN` | The full ARN of the ECS Task from which the log originated. | `No` | ### Configuration file @@ -94,15 +94,15 @@ pipeline: Skip_Long_Lines On Refresh_Interval 10 Rotate_Wait 30 - storage.type filesystem + Storage.Type filesystem Read_From_Head Off [FILTER] Name ecs Match * - ecs_tag_prefix ecs.var.lib.docker.containers. - add ecs_task_id $TaskID - add cluster $ClusterName + Ecs_Tag_Prefix ecs.var.lib.docker.containers. + Add ecs_task_id $TaskID + Add cluster $ClusterName [OUTPUT] Name stdout @@ -176,14 +176,14 @@ pipeline: Skip_Long_Lines On Refresh_Interval 10 Rotate_Wait 30 - storage.type filesystem + Storage.Type filesystem Read_From_Head Off [FILTER] Name ecs Match * - ecs_tag_prefix ecs.var.lib.docker.containers. - add resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName + Ecs_Tag_Prefix ecs.var.lib.docker.containers. + Add resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName [OUTPUT] Name stdout @@ -253,15 +253,15 @@ pipeline: Skip_Long_Lines On Refresh_Interval 10 Rotate_Wait 30 - storage.type filesystem + Storage.Type filesystem # Collect all logs on instance Read_From_Head On [FILTER] Name ecs Match * - cluster_metadata_only on - add cluster $ClusterName + Cluster_Metadata_Only on + Add cluster $ClusterName [OUTPUT] Name stdout diff --git a/pipeline/filters/geoip2-filter.md b/pipeline/filters/geoip2-filter.md index ae83e320d..d0736dd26 100644 --- a/pipeline/filters/geoip2-filter.md +++ b/pipeline/filters/geoip2-filter.md @@ -57,7 +57,7 @@ pipeline: Name geoip2 Match * Database GeoLite2-City.mmdb - Lookup_key remote_addr + Lookup_Key remote_addr Record country remote_addr %{country.names.en} Record isocode remote_addr %{country.iso_code} @@ -69,7 +69,7 @@ pipeline: {% endtab %} {% endtabs %} -Each `Record` parameter specifies the following triplet: +Each `record` parameter specifies the following triplet: - `country`: The field name to be added to records. - `remote_addr`: The lookup key to process. diff --git a/pipeline/filters/kubernetes.md b/pipeline/filters/kubernetes.md index a67f2683f..0c5b9fa99 100644 --- a/pipeline/filters/kubernetes.md +++ b/pipeline/filters/kubernetes.md @@ -164,8 +164,8 @@ The following annotations are available: | Annotation | Description | Default | | :--- | :--- | :--- | -| `fluentbit.io/parser[_stream][-container]` | Suggest a pre-defined parser. The parser must be registered already by Fluent Bit. This option will only be processed if Fluent Bit configuration (Kubernetes Filter) has enabled the option `K8S-Logging.Parser`. If present, the stream (stdout or stderr) will restrict that specific stream. If present, the container can override a specific container in a Pod. | _none_ | | `fluentbit.io/exclude[_stream][-container]` | Define whether to request that Fluent Bit excludes the logs generated by the pod. This option will be processed only if the Fluent Bit configuration (Kubernetes Filter) has enabled the option `K8S-Logging.Exclude`. | `False` | +| `fluentbit.io/parser[_stream][-container]` | Suggest a pre-defined parser. The parser must be registered already by Fluent Bit. This option will only be processed if Fluent Bit configuration (Kubernetes Filter) has enabled the option `K8S-Logging.Parser`. If present, the stream (stdout or stderr) will restrict that specific stream. If present, the container can override a specific container in a Pod. | _none_ | ### Annotation examples in pod definition diff --git a/pipeline/filters/log_to_metrics.md b/pipeline/filters/log_to_metrics.md index f0b425bf0..76dabcd26 100644 --- a/pipeline/filters/log_to_metrics.md +++ b/pipeline/filters/log_to_metrics.md @@ -85,8 +85,8 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + Flush 1 + Log_Level info [INPUT] Name dummy @@ -99,18 +99,18 @@ pipeline: Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode counter - metric_name count_all_dummy_messages - metric_description This metric counts dummy messages + Name log_to_metrics + Match dummy.log* + Tag test_metric + Metric_Mode counter + Metric_Name count_all_dummy_messages + Metric_Description This metric counts dummy messages [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + Name prometheus_exporter + Match * + Host 0.0.0.0 + Port 9999 ``` {% endtab %} @@ -193,8 +193,8 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + Flush 1 + Log_Level info [INPUT] Name dummy @@ -207,24 +207,24 @@ pipeline: Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode gauge - metric_name current_duration - metric_description This metric shows the current duration - value_field duration - kubernetes_mode on - regex message .*el.* - add_label app $kubernetes['labels']['app'] - label_field color - label_field shape + Name log_to_metrics + Match dummy.log* + Tag test_metric + Metric_Mode gauge + Metric_Name current_duration + Metric_Description This metric shows the current duration + Value_Field duration + Kubernetes_Mode on + Regex message .*el.* + Add_Label app $kubernetes['labels']['app'] + Label_Field color + Label_Field shape [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + Name prometheus_exporter + Match * + Host 0.0.0.0 + Port 9999 ``` {% endtab %} @@ -317,8 +317,8 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + Flush 1 + Log_Level info [INPUT] Name dummy @@ -331,24 +331,24 @@ pipeline: Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode histogram - metric_name current_duration - metric_description This metric shows the request duration - value_field duration - kubernetes_mode on - regex message .*el.* - add_label app $kubernetes['labels']['app'] - label_field color - label_field shape + Name log_to_metrics + Match dummy.log* + Tag test_metric + Metric_Mode histogram + Metric_Name current_duration + Metric_Description This metric shows the request duration + Value_Field duration + Kubernetes_Mode on + Regex message .*el.* + Add_Label app $kubernetes['labels']['app'] + Label_Field color + Label_Field shape [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + Name prometheus_exporter + Match * + Host 0.0.0.0 + Port 9999 ``` {% endtab %} @@ -464,8 +464,8 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + Flush 1 + Log_Level info [INPUT] Name dummy @@ -478,31 +478,31 @@ pipeline: Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode histogram - metric_name current_duration - metric_description This metric shows the HTTP request duration as histogram in milliseconds - value_field duration - kubernetes_mode on - bucket 1 - bucket 5 - bucket 10 - bucket 50 - bucket 100 - bucket 250 - bucket 500 - bucket 1000 - regex message .*el.* - label_field color - label_field shape + Name log_to_metrics + Match dummy.log* + Tag test_metric + Metric_Mode histogram + Metric_Name current_duration + Metric_Description This metric shows the HTTP request duration as histogram in milliseconds + Value_Field duration + Kubernetes_Mode on + Bucket 1 + Bucket 5 + Bucket 10 + Bucket 50 + Bucket 100 + Bucket 250 + Bucket 500 + Bucket 1000 + Regex message .*el.* + Label_Field color + Label_Field shape [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + Name prometheus_exporter + Match * + Host 0.0.0.0 + Port 9999 ``` {% endtab %} diff --git a/pipeline/filters/lua.md b/pipeline/filters/lua.md index c6b772346..04443ec44 100644 --- a/pipeline/filters/lua.md +++ b/pipeline/filters/lua.md @@ -70,8 +70,8 @@ pipeline: [FILTER] Name lua Match * - script test.lua - call cb_print + Script test.lua + Call cb_print [OUTPUT] Name null @@ -316,9 +316,9 @@ pipeline: ```text [SERVICE] - flush 1 - daemon off - log_level debug + Flush 1 + Daemon off + Log_Level debug [INPUT] Name random @@ -329,7 +329,7 @@ pipeline: Name lua Match * Call append_tag - code function append_tag(tag, timestamp, record) new_record = record new_record["tag"] = tag return 1, timestamp, new_record end + Code function append_tag(tag, timestamp, record) new_record = record new_record["tag"] = tag return 1, timestamp, new_record end [OUTPUT] Name stdout @@ -556,7 +556,7 @@ pipeline: [INPUT] Name tail Path /var/log/containers/*_istio-proxy-*.log - multiline.parser docker, cri + Multiline.Parser docker, cri Tag istio.* Mem_Buf_Limit 64MB Skip_Long_Lines Off diff --git a/pipeline/filters/nest.md b/pipeline/filters/nest.md index 751ceb7cb..96a9ad2c2 100644 --- a/pipeline/filters/nest.md +++ b/pipeline/filters/nest.md @@ -7,7 +7,7 @@ The _Nest_ filter plugin lets you operate on or with nested data. Its modes of o ## Example usage for `nest` -As an example using JSON notation, to nest keys matching the `Wildcard` value `Key*` under a new key `NestKey` the transformation becomes: +As an example using JSON notation, to nest keys matching the `wildcard` value `Key*` under a new key `NestKey` the transformation becomes: Input: @@ -33,7 +33,7 @@ Output: ## Example usage for `lift` -As an example using JSON notation, to lift keys nested under the `Nested_under` value `NestKey*` the transformation becomes: +As an example using JSON notation, to lift keys nested under the `nested_under` value `NestKey*` the transformation becomes: Input: @@ -63,12 +63,12 @@ The plugin supports the following configuration parameters: | Key | Value format | Operation | Description | | :--- | :--- | :--- | :--- | -| `Operation` | Enum [`nest` or `lift`] | | Select the operation `nest` or `lift` | -| `Wildcard` | Field wildcard | `nest` | Nest records which field matches the wildcard | -| `Nest_under` | Field string | `nest` | Nest records matching the `Wildcard` under this key | -| `Nested_under` | Field string | `lift` | Lift records nested under the `Nested_under` key | -| `Add_prefix` | Field string | Any | Prefix affected keys with this string | -| `Remove_prefix` | Field string | Any | Remove prefix from affected keys if it matches this string | +| `add_prefix` | Field string | Any | Prefix affected keys with this string | +| `nest_under` | Field string | `nest` | Nest records matching the `wildcard` under this key | +| `nested_under` | Field string | `lift` | Lift records nested under the `nested_under` key | +| `operation` | Enum [`nest` or `lift`] | | Select the operation `nest` or `lift` | +| `remove_prefix` | Field string | Any | Remove prefix from affected keys if it matches this string | +| `wildcard` | Field wildcard | `nest` | Nest records which field matches the wildcard | ## Get started diff --git a/pipeline/filters/nightfall.md b/pipeline/filters/nightfall.md index 5bb6e4b76..1757f956d 100644 --- a/pipeline/filters/nightfall.md +++ b/pipeline/filters/nightfall.md @@ -50,17 +50,17 @@ pipeline: ```text [INPUT] - name http - host 0.0.0.0 - port 8000 + Name http + Host 0.0.0.0 + Port 8000 [FILTER] Name nightfall Match * - nightfall_api_key - policy_id 5991946b-1cc8-4c38-9240-72677029a3f7 - sampling_rate 1 - tls.ca_path /etc/ssl/certs + Nightfall_Api_Key + Policy_Id 5991946b-1cc8-4c38-9240-72677029a3f7 + Sampling_Rate 1 + Tls.Ca_Path /etc/ssl/certs [OUTPUT] Name stdout diff --git a/pipeline/filters/parser.md b/pipeline/filters/parser.md index 1707a0f28..b654edf37 100644 --- a/pipeline/filters/parser.md +++ b/pipeline/filters/parser.md @@ -8,10 +8,10 @@ The plugin supports the following configuration parameters: | Key | Description | Default | | :--- | :--- | :--- | -| `Key_Name` | Specify field name in record to parse. | _none_ | -| `Parser` | Specify the parser name to interpret the field. Multiple parser entries are allowed (one per line). | _none_ | -| `Preserve_Key` | Keep the original `Key_Name` field in the parsed result. If false, the field will be removed. | `False` | -| `Reserve_Data` | Keep all other original fields in the parsed result. If false, all other original fields will be removed. | `False` | +| `key_name` | Specify field name in record to parse. | _none_ | +| `parser` | Specify the parser name to interpret the field. Multiple parser entries are allowed (one per line). | _none_ | +| `preserve_key` | Keep the original `key_name` field in the parsed result. If false, the field will be removed. | `false` | +| `reserve_data` | Keep all other original fields in the parsed result. If false, all other original fields will be removed. | `false` | ## Get started @@ -118,7 +118,7 @@ You can see the records `{"data":"100 0.5 true This is example"}` are parsed. By default, the parser plugin only keeps the parsed fields in its output. -If you enable `Reserve_Data`, all other fields are preserved. First the contents of the corresponding parsers file, depending on the choice for YAML or classic configurations, would be as follows: +If you enable `reserve_data`, all other fields are preserved. First the contents of the corresponding parsers file, depending on the choice for YAML or classic configurations, would be as follows: {% tabs %} {% tab title="parsers.yaml" %} @@ -143,7 +143,7 @@ parsers: {% endtab %} {% endtabs %} -Now add `Reserve_Data` to the filter section of the corresponding configuration file as follows: +Now add `reserve_data` to the filter section of the corresponding configuration file as follows: {% tabs %} {% tab title="fluent-bit.yaml" %} @@ -212,7 +212,7 @@ $ fluent-bit --config fluent-bit.conf [0] dummy.data: [[1750325240.682903000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "key1"=>"value1", "key2"=>"value2"}] ``` -If you enable `Reserve_Data` and `Preserve_Key`, the original key field will also be preserved. First the contents of the corresponding parsers file, depending on the choice for YAML or classic configurations, would be as follows: +If you enable `reserve_data` and `preserve_key`, the original key field will also be preserved. First the contents of the corresponding parsers file, depending on the choice for YAML or classic configurations, would be as follows: {% tabs %} {% tab title="parsers.yaml" %} @@ -237,7 +237,7 @@ parsers: {% endtab %} {% endtabs %} -Now add `Reserve_Data` and `Preserve_Key`to the filter section of the corresponding configuration file as follows: +Now add `reserve_data` and `preserve_key` to the filter section of the corresponding configuration file as follows: {% tabs %} {% tab title="fluent-bit.yaml" %} diff --git a/pipeline/filters/sysinfo.md b/pipeline/filters/sysinfo.md index 750247348..9481dc6c1 100644 --- a/pipeline/filters/sysinfo.md +++ b/pipeline/filters/sysinfo.md @@ -53,8 +53,8 @@ pipeline: [FILTER] Name sysinfo Match * - fluentbit_version_key flb_ver - os_name_key os_name + Fluentbit_Version_Key flb_ver + Os_Name_Key os_name [OUTPUT] Name stdout diff --git a/pipeline/filters/tensorflow.md b/pipeline/filters/tensorflow.md index 7df68d3d4..73fefb591 100644 --- a/pipeline/filters/tensorflow.md +++ b/pipeline/filters/tensorflow.md @@ -109,10 +109,10 @@ pipeline: [FILTER] Name tensorflow Match mqtt.data - input_field image - model_file /home/m/model.tflite - include_input_fields false - normalization_value 255 + Input_Field image + Model_File /home/m/model.tflite + Include_Input_Fields false + Normalization_Value 255 [OUTPUT] Name stdout diff --git a/pipeline/filters/type-converter.md b/pipeline/filters/type-converter.md index 6d03cb21c..b94dbc9d4 100644 --- a/pipeline/filters/type-converter.md +++ b/pipeline/filters/type-converter.md @@ -68,9 +68,9 @@ pipeline: [FILTER] Name type_converter Match * - uint_key Mem.total Mem.total_str string - uint_key Mem.used Mem.used_str string - uint_key Mem.free Mem.free_str string + Uint_Key Mem.total Mem.total_str string + Uint_Key Mem.used Mem.used_str string + Uint_Key Mem.free Mem.free_str string [OUTPUT] Name stdout