diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..6ea50b3 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,12 @@ +FROM fluent/fluentd:latest + +ENV DEBIAN_FRONTEND=noninteractive +USER root + +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + curl + +WORKDIR /workspaces + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..8d64e30 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,21 @@ +// .devcontainer/devcontainer.json +{ + "name": "Fluent Plugin Datadog Dev", + "build": { + "dockerfile": "Dockerfile" + }, + "workspaceFolder": "/workspaces/fluent-plugin-datadog", + "postCreateCommand": "bundle install --jobs 4 --retry 3", + "customizations": { + "vscode": { + "extensions": [ + "shopify.ruby-lsp", + "ms-azuretools.vscode-docker" + ], + "settings": { + "rubyLsp.bundleExec": true, + "rubyLsp.useBundler": true + } + } + } +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..119fce9 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,11 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "ruby_lsp", + "request": "launch", + "name": "Run Tests", + "program": "bundle exec ruby -Itest test/plugin/test_out_datadog.rb", + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 55c1fba..d46bc2a 100644 --- a/README.md +++ b/README.md @@ -156,3 +156,11 @@ To build a new version of this plugin and push it to RubyGems: `curl -u https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials`, it will ask for your password. +## Development Environment + +This repository includes the files to run it in a dev container in VS Code. To use it: + +1. Install VS Code and Docker. +2. Open the project in VS Code with the Dev Containers extension. + +VS Code will build and start the container automatically. diff --git a/fluent-plugin-datadog.gemspec b/fluent-plugin-datadog.gemspec index 1ab8110..f636f0e 100644 --- a/fluent-plugin-datadog.gemspec +++ b/fluent-plugin-datadog.gemspec @@ -28,8 +28,9 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", "~> 2.1" spec.add_development_dependency "test-unit", '~> 3.1' - spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency 'webmock', "~> 3.6.0" + spec.add_development_dependency "ruby-debug-ide" + spec.add_development_dependency "rake", "~> 12.0" spec.metadata = { 'bug_tracker_uri' => 'https://github.com/DataDog/fluent-plugin-datadog/issues', diff --git a/test/TestApp/README.md b/test/TestApp/README.md new file mode 100644 index 0000000..e1b5dd9 --- /dev/null +++ b/test/TestApp/README.md @@ -0,0 +1,47 @@ +# TestApp - Typical Fluentd Usage Example + +This directory demonstrates an example of fluentd usage with the Datadog plugin - using configuration files and running Fluentd as a service. + +## Files + +- **`fluent.conf`** - Fluentd configuration file +- **`start_fluentd.sh`** - Script to start Fluentd with the configuration +- **`send_test_logs.sh`** - Script to send test logs via HTTP (bash) + +## Quick Start + +### 1. Set your Datadog API Key + +```bash +export DD_API_KEY=your_api_key_here +``` + +### 2. Start Fluentd + +```bash +./start_fluentd.sh +``` + +This starts Fluentd as a service with the configuration file. Fluentd will: +- Listen on HTTP port 8888 for log ingestion +- Listen on Forward port 24224 for Fluentd protocol +- Route logs matching `test.**` to Datadog + +### 3. Send Test Logs + +In another terminal: + +```bash +# Using bash script +./send_test_logs.sh + +# Or manually with curl +curl -X POST -d 'json={"message":"Hello from Fluentd"}' \ + http://localhost:8888/test.app +``` + +### 4. Verify Logs + +Check your Datadog dashboard to see the logs appear. They should include: +- Original log fields +- Datadog metadata: `ddsource`, `ddtags`, `service`, `hostname`, `tag`, `@timestamp` diff --git a/test/TestApp/fluent.conf b/test/TestApp/fluent.conf new file mode 100644 index 0000000..098485b --- /dev/null +++ b/test/TestApp/fluent.conf @@ -0,0 +1,38 @@ +# Typical Fluentd configuration file for Datadog plugin +# This is how Fluentd is typically configured in production + +# HTTP input - allows sending logs via HTTP POST + + @type http + port 8888 + bind 0.0.0.0 + + +# Forward input - allows sending logs via Fluentd forward protocol + + @type forward + port 24224 + bind 0.0.0.0 + + +# Match events and send them to Datadog + + @type datadog + @id datadog_output + api_key YOUR_API_KEY_HERE + + # Optional configuration + include_tag_key true + tag_key 'tag' + service 'fluentd-test-app' + dd_source 'ruby' + dd_tags 'env:test,app:testapp' + + + @type memory + flush_interval 3s + chunk_limit_size 5m + chunk_limit_records 500 + + + diff --git a/test/TestApp/send_test_logs.sh b/test/TestApp/send_test_logs.sh new file mode 100755 index 0000000..79076e2 --- /dev/null +++ b/test/TestApp/send_test_logs.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2018 Datadog, Inc. + +# Script to send test logs to Fluentd via HTTP +# This demonstrates typical usage - sending logs to Fluentd's HTTP endpoint + +FLUENTD_URL="${FLUENTD_URL:-http://localhost:8888}" +TAG="${TAG:-test.app}" + +echo "Sending test logs to Fluentd at ${FLUENTD_URL}" +echo "Tag: ${TAG}" +echo "" + +# Send test log 1 +echo "Sending log 1: Test message" +curl -X POST -d 'json={"message":"Test log message from TestApp","level":"info","user":"test_user","action":"test_action"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" + +# Send test log 2 +echo "Sending log 2: Debug message" +curl -X POST -d 'json={"message":"Another test message","level":"debug","component":"test_component","status":"success"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" + +# Send test log 3 +echo "Sending log 3: Error simulation" +curl -X POST -d 'json={"message":"Error simulation","level":"error","error_code":"TEST_ERROR","stack_trace":"test_stack_trace"}' \ + "${FLUENTD_URL}/${TAG}" + +echo "" +echo "" +echo "Check Fluentd output or Datadog dashboard to verify logs were received." + diff --git a/test/TestApp/start_fluentd.sh b/test/TestApp/start_fluentd.sh new file mode 100755 index 0000000..6933e26 --- /dev/null +++ b/test/TestApp/start_fluentd.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2018 Datadog, Inc. + +# Script to start Fluentd with the Datadog plugin configuration +# This demonstrates typical Fluentd usage - running as a service with a config file + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/fluent.conf" + +# Check if API key is set +if [ -z "$DD_API_KEY" ]; then + echo "Warning: DD_API_KEY environment variable is not set." + echo "Please set it before starting Fluentd:" + echo " export DD_API_KEY=your_api_key_here" + echo "" + echo "Or edit fluent.conf and replace YOUR_API_KEY_HERE with your actual API key" + echo "" + if [ -t 0 ]; then + # Only prompt if running interactively + read -p "Continue anyway? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + else + echo "Non-interactive mode: Continuing with placeholder API key" + fi +fi + +# Replace API key in config if DD_API_KEY is set +if [ -n "$DD_API_KEY" ]; then + TEMP_CONFIG=$(mktemp) + sed "s/YOUR_API_KEY_HERE/$DD_API_KEY/g" "$CONFIG_FILE" > "$TEMP_CONFIG" + CONFIG_FILE="$TEMP_CONFIG" + trap "rm -f $TEMP_CONFIG" EXIT +fi + +echo "Starting Fluentd with configuration: ${SCRIPT_DIR}/fluent.conf" +echo "HTTP endpoint: http://localhost:8888" +echo "Forward endpoint: localhost:24224" +echo "" +echo "Press Ctrl+C to stop Fluentd" +echo "" + +# Start Fluentd +bundle exec fluentd -c "$CONFIG_FILE" -v +