diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c1d1c30 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Ignore everything +* + +# Allow files and directories +!Gemfile +!Gemfile.lock +!fluent.conf +!fluent.yaml +!entrypoint.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..43580d3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# All +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf +max_line_length = off + +# Markdown uses whitespace for formatting +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..314766e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..e69de29 diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..8d55e03 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + - package-ecosystem: docker + directory: / + schedule: + interval: daily + - package-ecosystem: bundler + directory: / + ignore: + - dependency-name: async + versions: [">=2"] + schedule: + interval: daily diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml new file mode 100644 index 0000000..379267d --- /dev/null +++ b/.github/workflows/commit.yaml @@ -0,0 +1,144 @@ +name: Commit + +on: + push: + branches: + - main + +jobs: + build: + name: Build OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + + - name: Install Cosign + uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b + + - name: Run Hadolint + id: hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + continue-on-error: true + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: false + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea + with: + flavor: | + latest=false + images: | + ghcr.io/${{ github.repository }} + docker.io/${{ github.repository }} + tags: | + type=raw,value=${{ matrix.os }}-main + type=raw,value=main,enable=${{ matrix.os == 'alpine' }} + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up QEMU + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b7e8507c6a3c89b7099a0198366d862c8f3ad8f1 + with: + image: "ghcr.io/${{ github.repository }}:${{ matrix.os }}-main" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + continue-on-error: true + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: true + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Fail workflow + if: ${{ steps.hadolint.outcome == 'failure' && steps.scan.outcome == 'failure' }} + run: | + set -euo pipefail + echo "::error::Code scanning failed." + exit 1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push OCI image + id: build_push + uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: false + push: true + + - name: Sign OCI image + env: + COSIGN_EXPERIMENTAL: true + run: | + set -euo pipefail + cosign sign --yes --recursive ghcr.io/${{ github.repository }}:main@${{ steps.build_push.outputs.digest }} + cosign sign --yes --recursive docker.io/${{ github.repository }}:main@${{ steps.build_push.outputs.digest }} diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..12d3645 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,91 @@ +name: Pull Request + +on: + pull_request: + branches: + - main + +jobs: + build: + name: Build OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: true + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea + with: + flavor: | + latest=false + images: | + ${{ github.repository }} + tags: | + type=raw,value=${{ matrix.os }}-local + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b7e8507c6a3c89b7099a0198366d862c8f3ad8f1 + with: + image: "${{ github.repository }}:${{ matrix.os }}-local" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: false + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..742c01e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,170 @@ +name: Release + +on: + push: + tags: + - v* + +jobs: + release: + name: Build & Release OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + + - name: Install Cosign + uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + continue-on-error: true + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: false + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea + with: + flavor: | + latest=false + images: | + ghcr.io/${{ github.repository }} + docker.io/${{ github.repository }} + tags: | + type=semver,pattern=${{ matrix.os }}-{{version}} + type=semver,pattern=${{ matrix.os }}-{{major}}.{{minor}} + type=semver,pattern=${{ matrix.os }}-{{major}} + type=raw,value=${{ matrix.os }} + type=semver,pattern={{version}},enable=${{ matrix.os == 'alpine' }} + type=semver,pattern={{major}}.{{minor}},enable=${{ matrix.os == 'alpine' }} + type=semver,pattern={{major}},enable=${{ matrix.os == 'alpine' }} + type=raw,value=latest,enable=${{ matrix.os == 'alpine' }} + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up QEMU + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b7e8507c6a3c89b7099a0198366d862c8f3ad8f1 + with: + image: "ghcr.io/${{ github.repository }}:${{ matrix.os }}-${{ steps.metadata.outputs.version }}" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + continue-on-error: true + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: true + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Fail workflow + if: ${{ steps.hadolint.outcome == 'failure' && steps.scan.outcome == 'failure' }} + run: | + set -euo pipefail + echo "::error::Code scanning failed." + exit 1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push OCI image + id: build_push + uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: false + push: true + + - name: Sign OCI image + env: + COSIGN_EXPERIMENTAL: true + run: | + set -euo pipefail + cosign sign --yes --recursive ghcr.io/${{ github.repository }}:${{ steps.metadata.outputs.version }}@${{ steps.build_push.outputs.digest }} + cosign sign --yes --recursive docker.io/${{ github.repository }}:${{ steps.metadata.outputs.version }}@${{ steps.build_push.outputs.digest }} + + - name: Update Docker repository description + uses: peter-evans/dockerhub-description@14881160433a81f9ebc838a253d4edd9e8fc35f1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ github.repository }} + + - name: Get changelog entry + id: changelog_reader + uses: mindsers/changelog-reader-action@17b72bf66059b7ac310abdc0b511948903e85ea5 + with: + path: ./CHANGELOG.md + version: v${{ steps.metadata.outputs.version }} + + - name: Create release + uses: ncipollo/release-action@4c75f0f2e4ae5f3c807cf0904605408e319dcaac + with: + token: ${{ secrets.GITHUB_TOKEN }} + allowUpdates: true + body: ${{ steps.changelog_reader.outputs.changes }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..65ea0e7 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,4 @@ +failure-threshold: warning +ignored: + - DL3008 + - DL3018 diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..a3c29f7 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,11 @@ +MD013: false +MD024: + siblings_only: true +MD028: false +MD033: + allowed_elements: + - br + - details + - summary + - sub + - sup diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a6a3d9c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Fluentd Aggregator Docker Image Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + + + +## [v0.0.1] - UNRELEASED + +### All Changes + +- Added initial version. diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..118ac45 --- /dev/null +++ b/Gemfile @@ -0,0 +1,22 @@ +source "https://rubygems.org" + +gem "oj", "3.13.21" +gem "json", "2.6.2" +gem "async", "1.30.3" +gem "async-http", "0.59.2" +gem "fluentd", "1.15.2" +gem "fluent-plugin-azure-loganalytics", "0.7.0" +gem "fluent-plugin-azurestorage-gen2", "0.3.3" +gem "fluent-plugin-cloudwatch-logs", "0.14.3" +gem "fluent-plugin-concat", "2.5.0" +gem "fluent-plugin-datadog", "0.14.2" +gem "fluent-plugin-elasticsearch", "5.2.4" +gem "fluent-plugin-grafana-loki", "1.2.18" +gem "fluent-plugin-kafka", "0.18.1" +gem "fluent-plugin-opensearch", "1.0.8" +gem "fluent-plugin-prometheus", "2.0.3" +gem "fluent-plugin-record-modifier", "2.1.1" +gem "fluent-plugin-rewrite-tag-filter", "2.4.0" +gem "fluent-plugin-route", "1.0.0" +gem "fluent-plugin-s3", "1.7.2" +gem "fluent-plugin-sqs", "3.0.0" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..f0797b9 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,253 @@ +GEM + remote: https://rubygems.org/ + specs: + async (1.30.3) + console (~> 1.10) + nio4r (~> 2.3) + timers (~> 4.1) + async-http (0.59.2) + async (>= 1.25) + async-io (>= 1.28) + async-pool (>= 0.2) + protocol-http (~> 0.23.1) + protocol-http1 (~> 0.14.0) + protocol-http2 (~> 0.14.0) + traces (>= 0.4.0) + async-io (1.34.0) + async + async-pool (0.3.12) + async (>= 1.25) + aws-eventstream (1.2.0) + aws-partitions (1.650.0) + aws-sdk-cloudwatchlogs (1.53.0) + aws-sdk-core (~> 3, >= 3.127.0) + aws-sigv4 (~> 1.1) + aws-sdk-core (3.164.0) + aws-eventstream (~> 1, >= 1.0.2) + aws-partitions (~> 1, >= 1.525.0) + aws-sigv4 (~> 1.1) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.58.0) + aws-sdk-core (~> 3, >= 3.127.0) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.116.0) + aws-sdk-core (~> 3, >= 3.127.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.4) + aws-sdk-sqs (1.51.1) + aws-sdk-core (~> 3, >= 3.127.0) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.5.2) + aws-eventstream (~> 1, >= 1.0.2) + azure-loganalytics-datacollector-api (0.5.0) + rest-client + concurrent-ruby (1.1.10) + connection_pool (2.3.0) + console (1.16.2) + fiber-local + cool.io (1.7.1) + digest-crc (0.6.4) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + elastic-transport (8.1.0) + faraday (< 3) + multi_json + elasticsearch (8.4.0) + elastic-transport (~> 8) + elasticsearch-api (= 8.4.0) + elasticsearch-api (8.4.0) + multi_json + ethon (0.15.0) + ffi (>= 1.15.0) + excon (0.93.1) + faraday (1.10.2) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.4) + multipart-post (~> 2) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + faraday_middleware-aws-sigv4 (0.6.1) + aws-sigv4 (~> 1.0) + faraday (>= 1.8, < 2) + ffi (1.15.5) + fiber-local (1.0.0) + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-azure-loganalytics (0.7.0) + azure-loganalytics-datacollector-api (>= 0.5.0) + fluentd (>= 0.14.15, < 2) + rest-client + yajl-ruby + fluent-plugin-azurestorage-gen2 (0.3.3) + concurrent-ruby (~> 1.1, >= 1.1.5) + fluentd (>= 1.0, < 2) + json (~> 2.1, >= 2.1.0) + typhoeus (~> 1.0, >= 1.0.1) + uuidtools (~> 2.1, >= 2.1.5) + yajl-ruby (~> 1.4) + fluent-plugin-cloudwatch-logs (0.14.3) + aws-sdk-cloudwatchlogs (~> 1.0) + fluentd (>= 1.8.0) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-datadog (0.14.2) + fluentd (>= 1, < 2) + net-http-persistent (~> 4.0.1) + fluent-plugin-elasticsearch (5.2.4) + elasticsearch + excon + faraday (~> 1.10) + fluentd (>= 0.14.22) + fluent-plugin-grafana-loki (1.2.18) + fluentd (>= 1.9.3, < 2) + fluent-plugin-kafka (0.18.1) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 1.4.0, < 2) + fluent-plugin-opensearch (1.0.8) + aws-sdk-core (~> 3) + excon + faraday (~> 1.10) + faraday_middleware-aws-sigv4 + fluentd (>= 0.14.22) + opensearch-ruby + fluent-plugin-prometheus (2.0.3) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-route (1.0.0) + fluentd (>= 0.14.0) + fluent-plugin-s3 (1.7.2) + aws-sdk-s3 (~> 1.60) + aws-sdk-sqs (~> 1.23) + fluentd (>= 0.14.22, < 2) + fluent-plugin-sqs (3.0.0) + aws-sdk-sqs (~> 1) + fluentd (>= 0.12.0, < 2) + yajl-ruby (~> 1.0) + fluentd (1.15.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.9.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.0, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) + yajl-ruby (~> 1.0) + http-accept (1.7.0) + http-cookie (1.0.5) + domain_name (~> 0.5) + http_parser.rb (0.8.0) + jmespath (1.6.1) + json (2.6.2) + ltsv (0.1.2) + mime-types (3.4.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2022.0105) + msgpack (1.6.0) + multi_json (1.15.0) + multipart-post (2.2.3) + net-http-persistent (4.0.1) + connection_pool (~> 2.2) + netrc (0.11.0) + nio4r (2.5.8) + oj (3.13.21) + opensearch-api (2.0.2) + multi_json + opensearch-ruby (2.0.3) + opensearch-api (= 2.0.2) + opensearch-transport (~> 2.0.0) + opensearch-transport (2.0.1) + faraday (>= 1.0, < 3) + multi_json + prometheus-client (4.0.0) + protocol-hpack (1.4.2) + protocol-http (0.23.12) + protocol-http1 (0.14.6) + protocol-http (~> 0.22) + protocol-http2 (0.14.2) + protocol-hpack (~> 1.4) + protocol-http (~> 0.18) + rake (13.0.6) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + ruby-kafka (1.5.0) + digest-crc + ruby2_keywords (0.0.5) + serverengine (2.3.0) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.5) + timers (4.3.5) + traces (0.7.0) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.5) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2022.5) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + uuidtools (2.2.0) + webrick (1.7.0) + yajl-ruby (1.4.3) + +PLATFORMS + x86_64-linux + x86_64-linux-musl + aarch64-linux + aarch64-linux-musl + +DEPENDENCIES + async (= 1.30.3) + async-http (= 0.59.2) + fluent-plugin-azure-loganalytics (= 0.7.0) + fluent-plugin-azurestorage-gen2 (= 0.3.3) + fluent-plugin-cloudwatch-logs (= 0.14.3) + fluent-plugin-concat (= 2.5.0) + fluent-plugin-datadog (= 0.14.2) + fluent-plugin-elasticsearch (= 5.2.4) + fluent-plugin-grafana-loki (= 1.2.18) + fluent-plugin-kafka (= 0.18.1) + fluent-plugin-opensearch (= 1.0.8) + fluent-plugin-prometheus (= 2.0.3) + fluent-plugin-record-modifier (= 2.1.1) + fluent-plugin-rewrite-tag-filter (= 2.4.0) + fluent-plugin-route (= 1.0.0) + fluent-plugin-s3 (= 1.7.2) + fluent-plugin-sqs (= 3.0.0) + fluentd (= 1.15.2) + json (= 2.6.2) + oj (= 3.13.21) + +BUNDLED WITH + 2.3.7 diff --git a/LICENSE b/LICENSE index 261eeb9..d9a10c0 100644 --- a/LICENSE +++ b/LICENSE @@ -174,28 +174,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9993ad5 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# Fluentd Aggregator Docker Image + +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/fluent/fluentd-aggregator-docker-image?sort=semver) +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/fluent/fluentd-aggregator?sort=semver)](https://hub.docker.com/r/fluent/fluentd-aggregator) +![linux](https://img.shields.io/badge/os-linux-brightgreen) +![amd64](https://img.shields.io/badge/arch-amd64-brightgreen) +![arm64](https://img.shields.io/badge/arch-arm64-brightgreen) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A [Fluentd](https://www.fluentd.org/) [OCI](https://opencontainers.org/) image to be used for log aggregation and based on the official [Fluentd Docker image](https://github.com/fluent/fluentd-docker-image) rebuilt as a multi-arch `linux/amd64` & `linux/arm64` image. + +## Aggregation Changes + +To optimise _Fluentd_ for log aggregation the default `fluent.conf` file has been overwritten to allow logs to be forwarded and printed to `stdout`, an additional directory `/fluentd/state` has been created, and plugins have also been added to support the aggregation role. + +### Plugins + +The following plugins have been added to the base image, to see the specific version please look in the [Gemfile](./Gemfile). + +- [fluent-plugin-azure-loganalytics](https://github.com/yokawasa/fluent-plugin-azure-loganalytics) +- [fluent-plugin-azurestorage-gen2](https://github.com/oleewere/fluent-plugin-azurestorage-gen2) +- [fluent-plugin-cloudwatch-logs](https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-logs) +- [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) +- [fluent-plugin-datadog](https://github.com/DataDog/fluent-plugin-datadog) +- [fluent-plugin-elasticsearch](https://docs.fluentd.org/output/elasticsearch) +- [fluent-plugin-grafana-loki](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd) +- [fluent-plugin-kafka](https://github.com/fluent/fluent-plugin-kafka) +- [fluent-plugin-opensearch](https://github.com/fluent/fluent-plugin-opensearch) +- [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) +- [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier) +- [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter) +- [fluent-plugin-route](https://github.com/tagomoris/fluent-plugin-route) +- [fluent-plugin-s3](https://docs.fluentd.org/output/s3) +- [fluent-plugin-sqs](https://github.com/ixixi/fluent-plugin-sqs) + +## Usage + +This image is available at [Docker Hub](https://hub.docker.com/r/fluent/fluentd-aggregator); the image version matches the _Fluentd_ version that it's based on. + +This image can be pulled with the following commands. + +```shell +docker pull fluent/fluentd-aggregator:latest + +docker pull ghcr.io/fluent/fluentd-aggregator:latest +``` + +This image can be tested by running the following command and then forwarding logs. + +```shell +docker run -p 24224:24224 fluent/fluentd-aggregator:latest +``` + +## License + +[Apache License, Version 2.0](./LICENSE). diff --git a/alpine.dockerfile b/alpine.dockerfile new file mode 100644 index 0000000..561fa87 --- /dev/null +++ b/alpine.dockerfile @@ -0,0 +1,45 @@ +FROM alpine:3.16.2 + +ENV FLUENTD_DISABLE_BUNDLER_INJECTION="1" + +COPY Gemfile Gemfile.lock /fluentd/ + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apk delete' has no effect +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc ruby-webrick \ + tini \ + libcurl \ + && apk add --no-cache --virtual .build-deps \ + build-base linux-headers \ + ruby-dev gnupg \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install bundler --version 2.3.7 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile \ + && gem sources --clear-all \ + && apk del .build-deps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup --system --gid 2000 fluent && adduser --system --ingroup fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent:fluent /fluentd + +COPY fluent.yaml /fluentd/etc/ +COPY --chmod=755 entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.yaml" + +ENV LD_PRELOAD="" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/debian.dockerfile b/debian.dockerfile new file mode 100644 index 0000000..39d3c78 --- /dev/null +++ b/debian.dockerfile @@ -0,0 +1,47 @@ +FROM ruby:3.1.2-slim-bullseye + +ENV FLUENTD_DISABLE_BUNDLER_INJECTION="1" + +COPY Gemfile Gemfile.lock /fluentd/ + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apt-get purge' has no effect +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates libjemalloc2 tini \ + && buildDeps=" \ + make gcc g++ libc-dev \ + wget bzip2 gnupg dirmngr \ + " \ + && apt-get install -y --no-install-recommends $buildDeps \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install bundler --version 2.3.7 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile \ + && gem sources --clear-all \ + && apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN groupadd --system --gid 2000 fluent && useradd --system --gid fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent:fluent /fluentd + +COPY fluent.conf /fluentd/etc/ +COPY --chmod=755 entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.conf" + +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d0f6b25 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep -e ' \-c' -e ' \-\-config' ; then + set -- "$@" --config /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep -e ' \-p' -e ' \-\-plugin' ; then + set -- "$@" --plugin /fluentd/plugins + fi +fi + +exec "$@" diff --git a/fluent.conf b/fluent.conf new file mode 100644 index 0000000..8a664b4 --- /dev/null +++ b/fluent.conf @@ -0,0 +1,18 @@ + + @type forward + @label @DEFAULT + port 24224 + bind 0.0.0.0 + + + + + diff --git a/fluent.yaml b/fluent.yaml new file mode 100644 index 0000000..ed857c7 --- /dev/null +++ b/fluent.yaml @@ -0,0 +1,23 @@ +system: + root_dir: /fluentd/state + +config: + - source: + $type: forward + $label: "@DEFAULT" + port: 24224 + bind: 0.0.0.0 + + - label: + $name: "@FLUENT_LOG" + config: + - match: + $tag: "fluent.*" + $type: stdout + + - label: + $name: "@DEFAULT" + config: + - match: + $tag: "**" + $type: stdout