Add OTLPMetricsWriter

.github/workflows/linux.bash

@@ -5,6 +5,7 @@ export PATH="/usr/lib/ccache/bin:/usr/lib/ccache:/usr/lib64/ccache:$PATH"
 export CCACHE_DIR=/icinga2/ccache
 export CTEST_OUTPUT_ON_FAILURE=1
 CMAKE_OPTS=()
+SCL_ENABLE_GCC=()
 # -Wstringop-overflow is notorious for false positives and has been a problem for years.
 # See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88443
 # -Wtemplate-id-cdtor leaks from using the generated headers. We should reenable this once
@@ -17,7 +18,7 @@ case "$DISTRO" in
     # - LibreSSL instead of OpenSSL 3 and
     # - no MariaDB or libpq as they depend on OpenSSL.
     # https://gitlab.alpinelinux.org/alpine/aports/-/blob/master/community/icinga2/APKBUILD
-    apk add bison boost-dev ccache cmake flex g++ libedit-dev libressl-dev ninja-build tzdata
+    apk add bison boost-dev ccache cmake flex g++ libedit-dev libressl-dev ninja-build tzdata protobuf-dev
     ln -vs /usr/lib/ninja-build/bin/ninja /usr/local/bin/ninja
     ;;
 
@@ -44,24 +45,24 @@ case "$DISTRO" in
 
   amazonlinux:20*)
     dnf install -y amazon-rpm-config bison cmake flex gcc-c++ ninja-build \
-      {boost,libedit,mariadb-connector-c,ncurses,openssl,postgresql,systemd}-devel
+      {boost,libedit,mariadb-connector-c,ncurses,openssl,postgresql,systemd,protobuf-lite}-devel
     ;;
 
   debian:*|ubuntu:*)
     apt-get update
     DEBIAN_FRONTEND=noninteractive apt-get install --no-install-{recommends,suggests} -y \
-      bison ccache cmake dpkg-dev flex g++ ninja-build tzdata \
-      lib{boost-all,edit,mariadb,ncurses,pq,ssl,systemd}-dev
+      bison ccache cmake dpkg-dev flex g++ ninja-build tzdata protobuf-compiler \
+      lib{boost-all,edit,mariadb,ncurses,pq,ssl,systemd,protobuf}-dev
     ;;
 
   fedora:*)
     dnf install -y bison ccache cmake flex gcc-c++ ninja-build redhat-rpm-config \
-      {boost,libedit,mariadb,ncurses,openssl,postgresql,systemd}-devel
+      {boost,libedit,mariadb,ncurses,openssl,postgresql,systemd,protobuf-lite}-devel
     ;;
 
   *suse*)
     zypper in -y bison ccache cmake flex gcc-c++ ninja rpm-config-SUSE \
-      {lib{edit,mariadb,openssl},ncurses,postgresql,systemd}-devel \
+      {lib{edit,mariadb,openssl},ncurses,postgresql,systemd,protobuf}-devel \
       libboost_{context,coroutine,filesystem,iostreams,program_options,regex,system,test,thread}-devel
     ;;
 
@@ -71,6 +72,10 @@ case "$DISTRO" in
     case "$DISTRO" in
       *:8)
         dnf config-manager --enable powertools
+        # Our Protobuf package on RHEL 8 is built with GCC 13, and since the ABI is not compatible with GCC 8,
+        # we need to enable the SCL repository and install the GCC 13 packages to be able to link against it.
+        SCL_ENABLE_GCC=(scl enable gcc-toolset-13 --)
+        dnf install -y gcc-toolset-13-gcc-c++ gcc-toolset-13-annobin-plugin-gcc
         ;;
       *)
         dnf config-manager --enable crb
@@ -79,6 +84,27 @@ case "$DISTRO" in
 
     dnf install -y bison ccache cmake gcc-c++ flex ninja-build redhat-rpm-config \
       {boost,bzip2,libedit,mariadb,ncurses,openssl,postgresql,systemd,xz,libzstd}-devel
+
+    # Rocky Linux 8 and 9 don't have a recent enough Protobuf compiler for OTel, so we need to add
+    # our repository to install the pre-built Protobuf devel package.
+    case "$DISTRO" in
+      *:[8-9])
+        rpm --import https://packages.icinga.com/icinga.key
+        cat > /etc/yum.repos.d/icinga-build-deps.repo <<'EOF'
+[icinga-build-deps]
+name=Icinga Build Dependencies
+baseurl=https://packages.icinga.com/build-dependencies/rhel/$releasever/release
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.icinga.com/icinga.key
+EOF
+        dnf install -y icinga-protobuf
+        # Tell CMake where to find our own Protobuf CMake config files.
+        CMAKE_OPTS+=(-DCMAKE_PREFIX_PATH="$(rpm -E '%{_libdir}')/icinga-protobuf/cmake")
+        ;;
+      *)
+        dnf install -y protobuf-lite-devel
+    esac
     ;;
 esac
 
@@ -96,8 +122,19 @@ case "$DISTRO" in
     source <(dpkg-buildflags --export=sh)
     export CFLAGS="${CFLAGS} ${WARN_FLAGS}"
     export CXXFLAGS="${CXXFLAGS} ${WARN_FLAGS}"
+
+    # The default Protobuf compiler is too old for OTel, so we need to turn it off on Debian 11 and Ubuntu 22.04.
+    case "$DISTRO" in
+      debian:11|ubuntu:22.04)
+        CMAKE_OPTS+=(-DICINGA2_WITH_OPENTELEMETRY=OFF)
+        ;;
+    esac
     ;;
   *)
+    # Turn off with OTel on Amazon Linux 2 as the default Protobuf compiler is way too old.
+    if [ "$DISTRO" = "amazonlinux:2" ]; then
+      CMAKE_OPTS+=(-DICINGA2_WITH_OPENTELEMETRY=OFF)
+    fi
     CMAKE_OPTS+=(-DCMAKE_{C,CXX}_FLAGS="$(rpm -E '%{optflags} %{?march_flag}') ${WARN_FLAGS}")
     export LDFLAGS="$(rpm -E '%{?build_ldflags}')"
     ;;
@@ -106,7 +143,7 @@ esac
 mkdir /icinga2/build
 cd /icinga2/build
 
-cmake \
+"${SCL_ENABLE_GCC[@]}" cmake \
   -GNinja \
   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
   -DICINGA2_UNITY_BUILD=ON \

CMakeLists.txt

@@ -37,6 +37,7 @@ option(ICINGA2_WITH_LIVESTATUS "Build the Livestatus module" ${ICINGA2_MASTER})
 option(ICINGA2_WITH_NOTIFICATION "Build the notification module" ON)
 option(ICINGA2_WITH_PERFDATA "Build the perfdata module" ${ICINGA2_MASTER})
 option(ICINGA2_WITH_ICINGADB "Build the IcingaDB module" ${ICINGA2_MASTER})
+option(ICINGA2_WITH_OPENTELEMETRY "Build the OpenTelemetry integration module" ${ICINGA2_MASTER})
 
 option (USE_SYSTEMD
  "Configure icinga as native systemd service instead of a SysV initscript" OFF)
@@ -207,6 +208,23 @@ set(HAVE_EDITLINE "${EDITLINE_FOUND}")
 find_package(Termcap)
 set(HAVE_TERMCAP "${TERMCAP_FOUND}")
 
+if(ICINGA2_WITH_OPENTELEMETRY)
+  # Newer Protobuf versions provide a CMake config package that we should prefer, since it implicitly
+  # links against all its dependencies (like absl, etc.) that would otherwise need to be linked manually.
+  # Thus, first try to find Protobuf in config mode and only fall back to module mode if that fails.
+  find_package(Protobuf CONFIG)
+  if(NOT Protobuf_FOUND)
+    # FindProtobuf.cmake in CMake versions < 3.31.0 is just broken and mixes up the Protobuf output directories
+    # and it doesn't even support to pass any PLUGIN_OPTIONS like "lite" to the protobuf_generate() function in
+    # order to generate code for the lite runtime without having to modify the proto files directly.
+    if(CMAKE_VERSION VERSION_LESS 3.31.0)
+      list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/third-party/cmake/protobuf")
+    endif()
+    find_package(Protobuf REQUIRED)
+  endif()
+  list(APPEND base_DEPS protobuf::libprotobuf-lite)
+endif()
+
 include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/lib
   ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib

Containerfile

@@ -38,6 +38,8 @@ RUN apt-get update && \
         libpq-dev \
         libssl-dev \
         libsystemd-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
         make && \
     rm -rf /var/lib/apt/lists/*
 
@@ -165,6 +167,7 @@ RUN apt-get update && \
         libmariadb3 \
         libmoosex-role-timer-perl \
         libpq5 \
+        libprotobuf-lite32t64 \
         libssl3 \
         libsystemd0 \
         mailutils \

doc/06-distributed-monitoring.md

@@ -2959,6 +2959,7 @@ By default, the following features provide advanced HA functionality:
 * [Graphite](09-object-types.md#objecttype-graphitewriter)
 * [InfluxDB](09-object-types.md#objecttype-influxdb2writer) (v1 and v2)
 * [OpenTsdb](09-object-types.md#objecttype-opentsdbwriter)
+* [OTLPMetrics](09-object-types.md#objecttype-otlpmetricswriter)
 * [Perfdata](09-object-types.md#objecttype-perfdatawriter) (for PNP)
 
 #### High-Availability with Checks <a id="distributed-monitoring-high-availability-checks"></a>

doc/09-object-types.md

@@ -1871,6 +1871,54 @@ Configuration Attributes:
   host_template             | Dictionary                | **Optional.** Specify additional tags to be included with host metrics. This requires a sub-dictionary named `tags`. Also specify a naming prefix by setting `metric`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags) and [OpenTSDB Metric Prefix](14-features.md#opentsdb-metric-prefix). More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`.
   service_template          | Dictionary                | **Optional.** Specify additional tags to be included with service metrics. This requires a sub-dictionary named `tags`. Also specify a naming prefix by setting `metric`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags) and [OpenTSDB Metric Prefix](14-features.md#opentsdb-metric-prefix). Defaults to an `empty Dictionary`.
 
+### OTLPMetricsWriter <a id="objecttype-otlpmetricswriter"></a>
+
+Emits metrics in [OpenTelemetry Protocol (OTLP)](https://opentelemetry.io/) format to a defined OpenTelemetry Collector
+or any other OTLP-compatible backend that accepts OTLP data over HTTP. This configuration object is available as
+[otlpmetrics feature](14-features.md#otlpmetrics-writer). You can find more information about OpenTelemetry and OTLP
+on the [OpenTelemetry website](https://opentelemetry.io/).
+
+A basic copy and pastable example configuration is shown below:
+
+```
+object OTLPMetricsWriter "otlp-metrics" {
+  host = "127.0.0.1"
+  port = 4318
+  metrics_endpoint = "/v1/metrics"
+  service_namespace = "icinga2-production"
+}
+```
+
+There are more configuration options available as described in the table below.
+
+| Name                          | Type       | Description                                                                                                                                  |
+|-------------------------------|------------|----------------------------------------------------------------------------------------------------------------------------------------------|
+| host                          | String     | **Required.** OTLP backend host address. Defaults to `127.0.0.1`.                                                                            |
+| port                          | Number     | **Required.** OTLP backend HTTP port. Defaults to `4318`.                                                                                    |
+| metrics\_endpoint             | String     | **Required.** OTLP metrics endpoint path. Defaults to `/v1/metrics`.                                                                         |
+| service\_namespace            | String     | **Required.** The namespace to associate with emitted metrics used in the `service.namespace` OTel resource attribute. Defaults to `icinga`. |
+| basic\_auth                   | Dictionary | **Optional.** Username and password for HTTP basic authentication.                                                                           |
+| host\_resource\_attributes    | Dictionary | **Optional.** Additional resource attributes to be included with host metrics. Defaults to none.                                             |
+| service\_resource\_attributes | Dictionary | **Optional.** Additional resource attributes to be included with service metrics. Defaults to none.                                          |
+| flush\_interval               | Duration   | **Optional.** How long to buffer data points before transferring to the OTLP backend. Defaults to `15s`.                                     |
+| flush\_threshold              | Number     | **Optional.** How many bytes to buffer before forcing a transfer to the OTLP backend. Defaults to `16MiB`.                                   |
+| enable\_ha                    | Boolean    | **Optional.** Enable the high availability functionality. Has no effect in non-cluster setups. Defaults to `true`.                           |
+| enable\_send\_thresholds      | Boolean    | **Optional.** Whether to stream warning, critical, minimum & maximum as separate metrics to the OTLP backend. Defaults to `false`.           |
+| diconnect\_timeout            | Duration   | **Optional.** Timeout to wait for any outstanding data to be flushed to the OTLP backend before disconnecting. Defaults to `10s`.            |
+| enable\_tls                   | Boolean    | **Optional.** Whether to use a TLS stream. Defaults to `false`.                                                                              |
+| tls\_insecure\_noverify       | Boolean    | **Optional.** Disable TLS peer verification. Defaults to `false`.                                                                            |
+| tls\_ca\_file                 | String     | **Optional.** Path to CA certificate to validate the remote host.                                                                            |
+| tls\_cert\_file               | String     | **Optional.** Path to the client certificate to present to the OTLP backend for mutual verification.                                         |
+| tls\_key\_file                | String     | **Optional.** Path to the client certificate key.                                                                                            |
+
+!!! tip
+
+    The `flush_threshold` is a byte size threshold, not a metric count threshold. By default, the writer will flush all
+    buffered metrics to the OTLP backend once the total size of buffered metrics exceeds 16 MiB. This number is chosen
+    based on the default `max_request_body_size` of the OpenTelemetry Collector, and you must adjust it according to the
+    `max_request_body_size` of your OTLP backend to avoid metrics being dropped due to exceeding the maximum request body
+    size. Furthermore, the writer may not flush at the exact byte size threshold due to the internal structure of OTLP
+    messages, so make sure that the threshold is lower than the configured `max_request_body_size` of your OTLP backend.
 
 ### PerfdataWriter <a id="objecttype-perfdatawriter"></a>