From 806fb2fb92e29c64cd0c519e446cbe9fa1d78ff6 Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Fri, 25 Apr 2025 16:33:11 -0400 Subject: [PATCH 01/10] Add devcontainer configuration --- .devcontainer/Dockerfile | 62 +++++++++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 53 ++++++++++++++++++++++++++++ .vscode/launch.json | 27 ++++++++++++++ .vscode/tasks.json | 54 ++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..b6a250e --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,62 @@ +# Use a Ruby image >= 3.1, as required by some dependencies like recent nokogiri +FROM ruby:3.2-slim-bookworm + +# Prevent interactive prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies required for building gems with native extensions +# and for general development. +# - build-essential: For compiling native extensions +# - git: For cloning repos if needed, although VS Code handles the main clone +# - curl, wget: Useful general tools +# - libxml2-dev, libxslt1-dev: Needed by nokogiri gem +# - zlib1g-dev: Needed by many gems +# - sqlite3, libsqlite3-dev: Needed by some gems that might be transitively included +# - libssl-dev: Needed for secure connections, potentially by some gems +# - pkg-config: Build helper +# - make: Used by the Makefile in the repo +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + curl \ + wget \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + sqlite3 \ + libsqlite3-dev \ + libssl-dev \ + pkg-config \ + make \ + zsh \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +# (Optional but Recommended) Create a non-root user for development +# This matches the default user setup by Dev Containers 'automatic' creation +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # Add the new user to the sudo group to allow them to run commands with sudo + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(ALL\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME \ + && rm -rf /var/lib/apt/lists/* + +# Set Zsh as the default shell for the non-root user +# Needs to be run as root, after the user is created and zsh is installed +RUN chsh -s $(which zsh) ${USERNAME} + +# Set the default working directory for subsequent commands and where the project code will be mounted +WORKDIR /workspaces + +# Switch to the non-root user +USER $USERNAME + +# The project code will be mounted into /workspaces by Dev Containers. +# Dependencies will be installed via postCreateCommand in devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..be25f55 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,53 @@ +// .devcontainer/devcontainer.json +{ + "name": "Fluent Plugin Datadog Dev", // A friendly name + // Specify the path to the custom Dockerfile + "build": { + "dockerfile": "Dockerfile" + }, + // Use the non-root user created in the Dockerfile + "remoteUser": "vscode", + // Set the workspace folder inside the container + "workspaceFolder": "/workspaces/fluent-plugin-datadog", + // Command to run after the container is created and the workspace is mounted. + // This installs the gem dependencies using Bundler. + "postCreateCommand": "bundle install && sh -c \"$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)\"", + // Forward ports that might be useful (e.g., for testing Fluentd) + // Fluentd's default forward port is 24224. The repo's docker-compose also uses this. + "forwardPorts": [ + 24224 + ], + // Configuration specific to VS Code + "customizations": { + "vscode": { + "settings": { + // Optional: Configure Ruby environment settings if needed + "ruby.useBundler": true, + "ruby.lint": { + "rubocop": true + }, + "ruby.format": "rubocop", + // Add other Ruby or general settings here + // Configure VS Code's integrated terminal to use Zsh + "terminal.integrated.defaultProfile.linux": "zsh (2)", + "terminal.integrated.profiles.linux": { + "zsh (2)": { + "path": "zsh" + } + } + }, + // Specify VS Code extensions to install automatically + "extensions": [ + "rebornix.ruby", // Official Ruby extension + "bungcip.better-toml", // For TOML config files + "kaiinui.vscode-ruby-test-explorer", // If you want a test explorer UI + "dbaeumer.vscode-eslint", // If there are JS parts or linting + "redhat.vscode-yaml", // For YAML files (like the docker-compose) + "ms-azuretools.vscode-docker" // Useful for working with Docker inside VS Code + // Add any other extensions relevant to your workflow + ] + } + } + // Uncomment to connect as root instead of the remoteUser. + // "remoteUser": "root" +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1a3d11d --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,27 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run Fluentd with Datadog Plugin", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/bin/fluentd", + "args": [ + "-c", + "${workspaceFolder}/test/fluentd.conf", + "-p", + "${workspaceFolder}/lib/fluent/plugin" + ], + "env": { + "FLUENTD_DISABLE_BUNDLER_INJECTION": "1" + } + }, + { + "name": "Run Tests", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/bin/rake", + "args": ["test"] + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..440595e --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,54 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Install Dependencies", + "type": "shell", + "command": "bundle install", + "group": "build", + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [] + }, + { + "label": "Build Gem", + "type": "shell", + "command": "gem build fluent-plugin-datadog.gemspec", + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [], + "dependsOn": ["Install Dependencies"] + }, + { + "label": "Run Tests", + "type": "shell", + "command": "bundle exec rake test", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [] + }, + { + "label": "Build and Test", + "dependsOn": [ + "Install Dependencies", + "Build Gem", + "Run Tests" + ], + "group": { + "kind": "build", + "isDefault": false + } + } + ] +} \ No newline at end of file From 2e768cbb4a922c79b98288a7782d408f85d66f36 Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:05:59 -0400 Subject: [PATCH 02/10] Fix launch.json and update readme --- .vscode/launch.json | 31 ++++++++++++++++++++----------- README.md | 8 ++++++++ fluent-plugin-datadog.gemspec | 2 ++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1a3d11d..b08c771 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,26 +2,35 @@ "version": "0.2.0", "configurations": [ { - "name": "Run Fluentd with Datadog Plugin", - "type": "node", + "name": "Debug Fluentd with Plugin", + "type": "Ruby", "request": "launch", - "program": "${workspaceFolder}/bin/fluentd", + "program": "/usr/local/bin/bundle", "args": [ + "exec", + "fluentd", "-c", - "${workspaceFolder}/test/fluentd.conf", + "${workspaceFolder}/test/fluentd-test.conf", "-p", "${workspaceFolder}/lib/fluent/plugin" ], - "env": { - "FLUENTD_DISABLE_BUNDLER_INJECTION": "1" - } + "cwd": "${workspaceFolder}", + "showDebuggerOutput": true, + "useBundler": true }, { - "name": "Run Tests", - "type": "node", + "name": "Debug Rake Tests", + "type": "Ruby", "request": "launch", - "program": "${workspaceFolder}/bin/rake", - "args": ["test"] + "program": "/usr/local/bin/bundle", + "args": [ + "exec", + "rake", + "test" + ], + "cwd": "${workspaceFolder}", + "showDebuggerOutput": true, + "useBundler": true } ] } \ No newline at end of file diff --git a/README.md b/README.md index 55c1fba..b05ba5c 100644 --- a/README.md +++ b/README.md @@ -156,3 +156,11 @@ To build a new version of this plugin and push it to RubyGems: `curl -u https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials`, it will ask for your password. +## Development Environment + +This repository includes the files to run it in a devcontainer in VS Code. To use it: + +1. Install VS Code and Docker +2. Open the project in VS Code with the Dev Containers extension. + +VS Code will build and start the container automatically. diff --git a/fluent-plugin-datadog.gemspec b/fluent-plugin-datadog.gemspec index 1ab8110..9a51ab1 100644 --- a/fluent-plugin-datadog.gemspec +++ b/fluent-plugin-datadog.gemspec @@ -30,6 +30,8 @@ Gem::Specification.new do |spec| spec.add_development_dependency "test-unit", '~> 3.1' spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency 'webmock', "~> 3.6.0" + spec.add_development_dependency "ruby-debug-ide" + spec.add_development_dependency "debase" spec.metadata = { 'bug_tracker_uri' => 'https://github.com/DataDog/fluent-plugin-datadog/issues', From aadecf71b4e71595417825689c728f262f81c1cf Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:23:23 -0400 Subject: [PATCH 03/10] CI fix --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93071b4..15d10d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,10 +14,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Configure Bundler to exclude development gems + run: bundle config set --local without development - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 3.0 + ruby-version: ${{ matrix.ruby-version }} bundler-cache: true - name: Run tests run: bundle exec rake From 852cba80fe94e1ccfb9940161d09f3d3cf8213c2 Mon Sep 17 00:00:00 2001 From: Lucas Liseth <36653792+soberpeach@users.noreply.github.com> Date: Mon, 12 May 2025 17:23:45 -0400 Subject: [PATCH 04/10] Update README.md Co-authored-by: Jen Gilbert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b05ba5c..66759f3 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ To build a new version of this plugin and push it to RubyGems: ## Development Environment -This repository includes the files to run it in a devcontainer in VS Code. To use it: +This repository includes the files to run it in a dev container in VS Code. To use it: 1. Install VS Code and Docker 2. Open the project in VS Code with the Dev Containers extension. From 3e8285024ec459937e95d27de866194798e3b217 Mon Sep 17 00:00:00 2001 From: Lucas Liseth <36653792+soberpeach@users.noreply.github.com> Date: Mon, 12 May 2025 17:23:52 -0400 Subject: [PATCH 05/10] Update README.md Co-authored-by: Jen Gilbert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 66759f3..d46bc2a 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ To build a new version of this plugin and push it to RubyGems: This repository includes the files to run it in a dev container in VS Code. To use it: -1. Install VS Code and Docker +1. Install VS Code and Docker. 2. Open the project in VS Code with the Dev Containers extension. VS Code will build and start the container automatically. From 7f76049fd100d3f9e49ef9616019d618bf8d2293 Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:25:38 -0400 Subject: [PATCH 06/10] CI fix --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15d10d3..0565d6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,12 +14,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Configure Bundler to exclude development gems - run: bundle config set --local without development - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true + - name: Configure Bundler to exclude development gems + run: bundle config set --local without development - name: Run tests run: bundle exec rake From 5523bedacd2418e12548d478cdbd35110728eca8 Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:28:50 -0400 Subject: [PATCH 07/10] More CI fixes --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0565d6c..5c07e37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,12 +14,17 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true + - name: Configure Bundler to exclude development gems run: bundle config set --local without development + + - name: Install dependencies + run: bundle install --jobs 4 --retry 3 + - name: Run tests run: bundle exec rake From df10cf5b0daadf833eeb7bb6f4aa2a285b4bf38e Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:30:54 -0400 Subject: [PATCH 08/10] Add rake as runtime dependency --- fluent-plugin-datadog.gemspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fluent-plugin-datadog.gemspec b/fluent-plugin-datadog.gemspec index 9a51ab1..f636f0e 100644 --- a/fluent-plugin-datadog.gemspec +++ b/fluent-plugin-datadog.gemspec @@ -28,10 +28,9 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", "~> 2.1" spec.add_development_dependency "test-unit", '~> 3.1' - spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency 'webmock', "~> 3.6.0" spec.add_development_dependency "ruby-debug-ide" - spec.add_development_dependency "debase" + spec.add_development_dependency "rake", "~> 12.0" spec.metadata = { 'bug_tracker_uri' => 'https://github.com/DataDog/fluent-plugin-datadog/issues', From cc00c4e6296ef9beff4363fadc7c2744f334ff67 Mon Sep 17 00:00:00 2001 From: Lucas Liseth Date: Mon, 12 May 2025 17:34:04 -0400 Subject: [PATCH 09/10] Add runtime dependency --- fluent-plugin-datadog.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-datadog.gemspec b/fluent-plugin-datadog.gemspec index f636f0e..f3a84f0 100644 --- a/fluent-plugin-datadog.gemspec +++ b/fluent-plugin-datadog.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "fluentd", [">= 1", "< 2"] spec.add_runtime_dependency "net-http-persistent", '~> 4.0.1' - + spec.add_development_dependency "bundler", "~> 2.1" spec.add_development_dependency "test-unit", '~> 3.1' spec.add_development_dependency 'webmock', "~> 3.6.0" From 7dd431247d54567f3ef9325fa68ccbd83e8599b3 Mon Sep 17 00:00:00 2001 From: Ryan Hall Date: Wed, 10 Dec 2025 17:37:25 -0500 Subject: [PATCH 10/10] finalize the developer environment experience --- .devcontainer/Dockerfile | 58 +++------------------------------ .devcontainer/devcontainer.json | 50 +++++----------------------- .github/workflows/test.yml | 11 ++----- .vscode/launch.json | 31 ++---------------- .vscode/tasks.json | 54 ------------------------------ fluent-plugin-datadog.gemspec | 2 +- test/TestApp/README.md | 47 ++++++++++++++++++++++++++ test/TestApp/fluent.conf | 38 +++++++++++++++++++++ test/TestApp/send_test_logs.sh | 41 +++++++++++++++++++++++ test/TestApp/start_fluentd.sh | 50 ++++++++++++++++++++++++++++ 10 files changed, 195 insertions(+), 187 deletions(-) delete mode 100644 .vscode/tasks.json create mode 100644 test/TestApp/README.md create mode 100644 test/TestApp/fluent.conf create mode 100755 test/TestApp/send_test_logs.sh create mode 100755 test/TestApp/start_fluentd.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b6a250e..6ea50b3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,62 +1,12 @@ -# Use a Ruby image >= 3.1, as required by some dependencies like recent nokogiri -FROM ruby:3.2-slim-bookworm +FROM fluent/fluentd:latest -# Prevent interactive prompts from apt ENV DEBIAN_FRONTEND=noninteractive +USER root -# Install system dependencies required for building gems with native extensions -# and for general development. -# - build-essential: For compiling native extensions -# - git: For cloning repos if needed, although VS Code handles the main clone -# - curl, wget: Useful general tools -# - libxml2-dev, libxslt1-dev: Needed by nokogiri gem -# - zlib1g-dev: Needed by many gems -# - sqlite3, libsqlite3-dev: Needed by some gems that might be transitively included -# - libssl-dev: Needed for secure connections, potentially by some gems -# - pkg-config: Build helper -# - make: Used by the Makefile in the repo -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y \ build-essential \ git \ - curl \ - wget \ - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ - sqlite3 \ - libsqlite3-dev \ - libssl-dev \ - pkg-config \ - make \ - zsh \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + curl - -# (Optional but Recommended) Create a non-root user for development -# This matches the default user setup by Dev Containers 'automatic' creation -ARG USERNAME=vscode -ARG USER_UID=1000 -ARG USER_GID=$USER_UID - -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - # Add the new user to the sudo group to allow them to run commands with sudo - && apt-get update \ - && apt-get install -y sudo \ - && echo $USERNAME ALL=\(ALL\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME \ - && rm -rf /var/lib/apt/lists/* - -# Set Zsh as the default shell for the non-root user -# Needs to be run as root, after the user is created and zsh is installed -RUN chsh -s $(which zsh) ${USERNAME} - -# Set the default working directory for subsequent commands and where the project code will be mounted WORKDIR /workspaces -# Switch to the non-root user -USER $USERNAME - -# The project code will be mounted into /workspaces by Dev Containers. -# Dependencies will be installed via postCreateCommand in devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index be25f55..8d64e30 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,53 +1,21 @@ // .devcontainer/devcontainer.json { - "name": "Fluent Plugin Datadog Dev", // A friendly name - // Specify the path to the custom Dockerfile + "name": "Fluent Plugin Datadog Dev", "build": { "dockerfile": "Dockerfile" }, - // Use the non-root user created in the Dockerfile - "remoteUser": "vscode", - // Set the workspace folder inside the container "workspaceFolder": "/workspaces/fluent-plugin-datadog", - // Command to run after the container is created and the workspace is mounted. - // This installs the gem dependencies using Bundler. - "postCreateCommand": "bundle install && sh -c \"$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)\"", - // Forward ports that might be useful (e.g., for testing Fluentd) - // Fluentd's default forward port is 24224. The repo's docker-compose also uses this. - "forwardPorts": [ - 24224 - ], - // Configuration specific to VS Code + "postCreateCommand": "bundle install --jobs 4 --retry 3", "customizations": { "vscode": { - "settings": { - // Optional: Configure Ruby environment settings if needed - "ruby.useBundler": true, - "ruby.lint": { - "rubocop": true - }, - "ruby.format": "rubocop", - // Add other Ruby or general settings here - // Configure VS Code's integrated terminal to use Zsh - "terminal.integrated.defaultProfile.linux": "zsh (2)", - "terminal.integrated.profiles.linux": { - "zsh (2)": { - "path": "zsh" - } - } - }, - // Specify VS Code extensions to install automatically "extensions": [ - "rebornix.ruby", // Official Ruby extension - "bungcip.better-toml", // For TOML config files - "kaiinui.vscode-ruby-test-explorer", // If you want a test explorer UI - "dbaeumer.vscode-eslint", // If there are JS parts or linting - "redhat.vscode-yaml", // For YAML files (like the docker-compose) - "ms-azuretools.vscode-docker" // Useful for working with Docker inside VS Code - // Add any other extensions relevant to your workflow - ] + "shopify.ruby-lsp", + "ms-azuretools.vscode-docker" + ], + "settings": { + "rubyLsp.bundleExec": true, + "rubyLsp.useBundler": true + } } } - // Uncomment to connect as root instead of the remoteUser. - // "remoteUser": "root" } \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c07e37..93071b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,17 +14,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: ${{ matrix.ruby-version }} - - - name: Configure Bundler to exclude development gems - run: bundle config set --local without development - - - name: Install dependencies - run: bundle install --jobs 4 --retry 3 - + ruby-version: 3.0 + bundler-cache: true - name: Run tests run: bundle exec rake diff --git a/.vscode/launch.json b/.vscode/launch.json index b08c771..119fce9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,35 +2,10 @@ "version": "0.2.0", "configurations": [ { - "name": "Debug Fluentd with Plugin", - "type": "Ruby", + "type": "ruby_lsp", "request": "launch", - "program": "/usr/local/bin/bundle", - "args": [ - "exec", - "fluentd", - "-c", - "${workspaceFolder}/test/fluentd-test.conf", - "-p", - "${workspaceFolder}/lib/fluent/plugin" - ], - "cwd": "${workspaceFolder}", - "showDebuggerOutput": true, - "useBundler": true - }, - { - "name": "Debug Rake Tests", - "type": "Ruby", - "request": "launch", - "program": "/usr/local/bin/bundle", - "args": [ - "exec", - "rake", - "test" - ], - "cwd": "${workspaceFolder}", - "showDebuggerOutput": true, - "useBundler": true + "name": "Run Tests", + "program": "bundle exec ruby -Itest test/plugin/test_out_datadog.rb", } ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 440595e..0000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "Install Dependencies", - "type": "shell", - "command": "bundle install", - "group": "build", - "presentation": { - "reveal": "always", - "panel": "new" - }, - "problemMatcher": [] - }, - { - "label": "Build Gem", - "type": "shell", - "command": "gem build fluent-plugin-datadog.gemspec", - "group": { - "kind": "build", - "isDefault": true - }, - "presentation": { - "reveal": "always", - "panel": "new" - }, - "problemMatcher": [], - "dependsOn": ["Install Dependencies"] - }, - { - "label": "Run Tests", - "type": "shell", - "command": "bundle exec rake test", - "group": "test", - "presentation": { - "reveal": "always", - "panel": "new" - }, - "problemMatcher": [] - }, - { - "label": "Build and Test", - "dependsOn": [ - "Install Dependencies", - "Build Gem", - "Run Tests" - ], - "group": { - "kind": "build", - "isDefault": false - } - } - ] -} \ No newline at end of file diff --git a/fluent-plugin-datadog.gemspec b/fluent-plugin-datadog.gemspec index f3a84f0..f636f0e 100644 --- a/fluent-plugin-datadog.gemspec +++ b/fluent-plugin-datadog.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "fluentd", [">= 1", "< 2"] spec.add_runtime_dependency "net-http-persistent", '~> 4.0.1' - + spec.add_development_dependency "bundler", "~> 2.1" spec.add_development_dependency "test-unit", '~> 3.1' spec.add_development_dependency 'webmock', "~> 3.6.0" diff --git a/test/TestApp/README.md b/test/TestApp/README.md new file mode 100644 index 0000000..e1b5dd9 --- /dev/null +++ b/test/TestApp/README.md @@ -0,0 +1,47 @@ +# TestApp - Typical Fluentd Usage Example + +This directory demonstrates an example of fluentd usage with the Datadog plugin - using configuration files and running Fluentd as a service. + +## Files + +- **`fluent.conf`** - Fluentd configuration file +- **`start_fluentd.sh`** - Script to start Fluentd with the configuration +- **`send_test_logs.sh`** - Script to send test logs via HTTP (bash) + +## Quick Start + +### 1. Set your Datadog API Key + +```bash +export DD_API_KEY=your_api_key_here +``` + +### 2. Start Fluentd + +```bash +./start_fluentd.sh +``` + +This starts Fluentd as a service with the configuration file. Fluentd will: +- Listen on HTTP port 8888 for log ingestion +- Listen on Forward port 24224 for Fluentd protocol +- Route logs matching `test.**` to Datadog + +### 3. Send Test Logs + +In another terminal: + +```bash +# Using bash script +./send_test_logs.sh + +# Or manually with curl +curl -X POST -d 'json={"message":"Hello from Fluentd"}' \ + http://localhost:8888/test.app +``` + +### 4. Verify Logs + +Check your Datadog dashboard to see the logs appear. They should include: +- Original log fields +- Datadog metadata: `ddsource`, `ddtags`, `service`, `hostname`, `tag`, `@timestamp` diff --git a/test/TestApp/fluent.conf b/test/TestApp/fluent.conf new file mode 100644 index 0000000..098485b --- /dev/null +++ b/test/TestApp/fluent.conf @@ -0,0 +1,38 @@ +# Typical Fluentd configuration file for Datadog plugin +# This is how Fluentd is typically configured in production + +# HTTP input - allows sending logs via HTTP POST + + @type http + port 8888 + bind 0.0.0.0 + + +# Forward input - allows sending logs via Fluentd forward protocol + + @type forward + port 24224 + bind 0.0.0.0 + + +# Match events and send them to Datadog + + @type datadog + @id datadog_output + api_key YOUR_API_KEY_HERE + + # Optional configuration + include_tag_key true + tag_key 'tag' + service 'fluentd-test-app' + dd_source 'ruby' + dd_tags 'env:test,app:testapp' + + + @type memory + flush_interval 3s + chunk_limit_size 5m + chunk_limit_records 500 + + + diff --git a/test/TestApp/send_test_logs.sh b/test/TestApp/send_test_logs.sh new file mode 100755 index 0000000..79076e2 --- /dev/null +++ b/test/TestApp/send_test_logs.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2018 Datadog, Inc. + +# Script to send test logs to Fluentd via HTTP +# This demonstrates typical usage - sending logs to Fluentd's HTTP endpoint + +FLUENTD_URL="${FLUENTD_URL:-http://localhost:8888}" +TAG="${TAG:-test.app}" + +echo "Sending test logs to Fluentd at ${FLUENTD_URL}" +echo "Tag: ${TAG}" +echo "" + +# Send test log 1 +echo "Sending log 1: Test message" +curl -X POST -d 'json={"message":"Test log message from TestApp","level":"info","user":"test_user","action":"test_action"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" + +# Send test log 2 +echo "Sending log 2: Debug message" +curl -X POST -d 'json={"message":"Another test message","level":"debug","component":"test_component","status":"success"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" + +# Send test log 3 +echo "Sending log 3: Error simulation" +curl -X POST -d 'json={"message":"Error simulation","level":"error","error_code":"TEST_ERROR","stack_trace":"test_stack_trace"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" +echo "Check Fluentd output or Datadog dashboard to verify logs were received." + diff --git a/test/TestApp/start_fluentd.sh b/test/TestApp/start_fluentd.sh new file mode 100755 index 0000000..6933e26 --- /dev/null +++ b/test/TestApp/start_fluentd.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2018 Datadog, Inc. + +# Script to start Fluentd with the Datadog plugin configuration +# This demonstrates typical Fluentd usage - running as a service with a config file + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/fluent.conf" + +# Check if API key is set +if [ -z "$DD_API_KEY" ]; then + echo "Warning: DD_API_KEY environment variable is not set." + echo "Please set it before starting Fluentd:" + echo " export DD_API_KEY=your_api_key_here" + echo "" + echo "Or edit fluent.conf and replace YOUR_API_KEY_HERE with your actual API key" + echo "" + if [ -t 0 ]; then + # Only prompt if running interactively + read -p "Continue anyway? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + else + echo "Non-interactive mode: Continuing with placeholder API key" + fi +fi + +# Replace API key in config if DD_API_KEY is set +if [ -n "$DD_API_KEY" ]; then + TEMP_CONFIG=$(mktemp) + sed "s/YOUR_API_KEY_HERE/$DD_API_KEY/g" "$CONFIG_FILE" > "$TEMP_CONFIG" + CONFIG_FILE="$TEMP_CONFIG" + trap "rm -f $TEMP_CONFIG" EXIT +fi + +echo "Starting Fluentd with configuration: ${SCRIPT_DIR}/fluent.conf" +echo "HTTP endpoint: http://localhost:8888" +echo "Forward endpoint: localhost:24224" +echo "" +echo "Press Ctrl+C to stop Fluentd" +echo "" + +# Start Fluentd +bundle exec fluentd -c "$CONFIG_FILE" -v +