From acfff6cb567808c0d0614aff2018583a1992f7ac Mon Sep 17 00:00:00 2001 From: bwplotka Date: Wed, 18 Feb 2026 08:48:42 +0000 Subject: [PATCH 01/13] rw2senderfix: skip all by default Signed-off-by: bwplotka --- remotewrite/sender/backoff_test.go | 4 +++- remotewrite/sender/batching_test.go | 7 +++++-- remotewrite/sender/combined_test.go | 4 +++- remotewrite/sender/edge_cases_test.go | 8 ++++++-- remotewrite/sender/error_handling_test.go | 8 ++++++-- remotewrite/sender/exemplars_test.go | 4 +++- remotewrite/sender/fallback_test.go | 8 ++++++-- remotewrite/sender/histograms_test.go | 4 +++- remotewrite/sender/labels_test.go | 4 +++- remotewrite/sender/metadata_test.go | 4 +++- remotewrite/sender/protocol_test.go | 8 ++++++-- remotewrite/sender/response_test.go | 8 ++++++-- remotewrite/sender/retry_test.go | 4 +++- remotewrite/sender/rw1_compat_test.go | 8 ++++++-- remotewrite/sender/samples_test.go | 8 ++++++-- remotewrite/sender/symbols_test.go | 8 ++++++-- remotewrite/sender/timestamps_test.go | 4 +++- 17 files changed, 77 insertions(+), 26 deletions(-) diff --git a/remotewrite/sender/backoff_test.go b/remotewrite/sender/backoff_test.go index bd56de35..18bff9f4 100644 --- a/remotewrite/sender/backoff_test.go +++ b/remotewrite/sender/backoff_test.go @@ -53,7 +53,9 @@ func (ttr *TimestampTrackingReceiver) GetTimestamps() []time.Time { } // TestBackoffBehavior validates backoff and exponential backoff implementation. -func TestBackoffBehavior(t *testing.T) { +func TestBackoffBehavior_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []struct { name string description string diff --git a/remotewrite/sender/batching_test.go b/remotewrite/sender/batching_test.go index d14e0f23..34355af9 100644 --- a/remotewrite/sender/batching_test.go +++ b/remotewrite/sender/batching_test.go @@ -23,7 +23,8 @@ import ( ) // TestBatchingBehavior validates sender batching and queueing behavior. -func TestBatchingBehavior(t *testing.T) { +func TestBatchingBehavior_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") tests := []TestCase{ { Name: "multiple_series_per_request", @@ -176,7 +177,9 @@ memory_usage_bytes 1048576 } // TestConcurrentRequests validates parallel request handling. -func TestConcurrentRequests(t *testing.T) { +func TestConcurrentRequests_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "MAY") t.Attr("description", "Sender MAY send multiple requests in parallel") diff --git a/remotewrite/sender/combined_test.go b/remotewrite/sender/combined_test.go index 359fe75a..637e7cd1 100644 --- a/remotewrite/sender/combined_test.go +++ b/remotewrite/sender/combined_test.go @@ -21,7 +21,9 @@ import ( ) // TestCombinedFeatures validates integration of multiple Remote Write 2.0 features. -func TestCombinedFeatures(t *testing.T) { +func TestCombinedFeatures_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "samples_with_metadata", diff --git a/remotewrite/sender/edge_cases_test.go b/remotewrite/sender/edge_cases_test.go index ad368a5f..19c3c8af 100644 --- a/remotewrite/sender/edge_cases_test.go +++ b/remotewrite/sender/edge_cases_test.go @@ -23,7 +23,9 @@ import ( ) // TestEdgeCases validates sender behavior in edge case scenarios. -func TestEdgeCases(t *testing.T) { +func TestEdgeCases_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "empty_scrape", @@ -360,7 +362,9 @@ rpc_duration_count 1000 } // TestRobustnessUnderLoad validates sender behavior under stress. -func TestRobustnessUnderLoad(t *testing.T) { +func TestRobustnessUnderLoad_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "SHOULD") t.Attr("description", "Sender SHOULD remain stable under load") diff --git a/remotewrite/sender/error_handling_test.go b/remotewrite/sender/error_handling_test.go index 419c1402..c518879f 100644 --- a/remotewrite/sender/error_handling_test.go +++ b/remotewrite/sender/error_handling_test.go @@ -23,7 +23,9 @@ import ( ) // TestErrorHandling validates sender error handling in various failure scenarios. -func TestErrorHandling(t *testing.T) { +func TestErrorHandling_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []struct { name string description string @@ -195,7 +197,9 @@ func TestErrorHandling(t *testing.T) { } // TestNetworkErrors validates handling of network-level errors. -func TestNetworkErrors(t *testing.T) { +func TestNetworkErrors_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "SHOULD") t.Attr("description", "Sender SHOULD handle network errors gracefully") diff --git a/remotewrite/sender/exemplars_test.go b/remotewrite/sender/exemplars_test.go index da8eec7e..51a42ae1 100644 --- a/remotewrite/sender/exemplars_test.go +++ b/remotewrite/sender/exemplars_test.go @@ -19,7 +19,9 @@ import ( ) // TestExemplarEncoding validates exemplar encoding in Remote Write 2.0. -func TestExemplarEncoding(t *testing.T) { +func TestExemplarEncoding_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "exemplar_with_trace_id", diff --git a/remotewrite/sender/fallback_test.go b/remotewrite/sender/fallback_test.go index 5f4b6fed..5ef33686 100644 --- a/remotewrite/sender/fallback_test.go +++ b/remotewrite/sender/fallback_test.go @@ -66,7 +66,9 @@ func (ftr *FallbackTrackingReceiver) ShouldReturn415() bool { } // TestFallbackBehavior validates RW 2.0 to RW 1.0 fallback on 415 response. -func TestFallbackBehavior(t *testing.T) { +func TestFallbackBehavior_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []struct { name string description string @@ -282,7 +284,9 @@ func TestFallbackBehavior(t *testing.T) { } // TestNoFallbackOn2xx validates that fallback doesn't happen on success. -func TestNoFallbackOn2xx(t *testing.T) { +func TestNoFallbackOn2xx_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "MUST") t.Attr("description", "Sender MUST NOT fallback when receiving 2xx success responses") diff --git a/remotewrite/sender/histograms_test.go b/remotewrite/sender/histograms_test.go index c9e99045..8873fcd7 100644 --- a/remotewrite/sender/histograms_test.go +++ b/remotewrite/sender/histograms_test.go @@ -18,7 +18,9 @@ import ( ) // TestHistogramEncoding validates native histogram encoding. -func TestHistogramEncoding(t *testing.T) { +func TestHistogramEncoding_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "native_histogram_structure", diff --git a/remotewrite/sender/labels_test.go b/remotewrite/sender/labels_test.go index c6b768dc..6536f349 100644 --- a/remotewrite/sender/labels_test.go +++ b/remotewrite/sender/labels_test.go @@ -20,7 +20,9 @@ import ( ) // TestLabelValidation validates label encoding and formatting. -func TestLabelValidation(t *testing.T) { +func TestLabelValidation_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "label_lexicographic_ordering", diff --git a/remotewrite/sender/metadata_test.go b/remotewrite/sender/metadata_test.go index 327c6314..acafe221 100644 --- a/remotewrite/sender/metadata_test.go +++ b/remotewrite/sender/metadata_test.go @@ -21,7 +21,9 @@ import ( ) // TestMetadataEncoding validates metric metadata encoding. -func TestMetadataEncoding(t *testing.T) { +func TestMetadataEncoding_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "metadata_type_counter", diff --git a/remotewrite/sender/protocol_test.go b/remotewrite/sender/protocol_test.go index 8bc6ee13..01569b6b 100644 --- a/remotewrite/sender/protocol_test.go +++ b/remotewrite/sender/protocol_test.go @@ -22,7 +22,9 @@ import ( ) // TestProtocolCompliance validates HTTP protocol requirements for Remote Write 2.0 senders. -func TestProtocolCompliance(t *testing.T) { +func TestProtocolCompliance_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "content_type_protobuf", @@ -130,7 +132,9 @@ func TestProtocolCompliance(t *testing.T) { } // TestHTTPMethod validates that senders use POST method for remote write. -func TestHTTPMethod(t *testing.T) { +func TestHTTPMethod_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "MUST") t.Attr("description", "Sender MUST use POST method for remote write requests") diff --git a/remotewrite/sender/response_test.go b/remotewrite/sender/response_test.go index 6d2b1a8c..d90c837f 100644 --- a/remotewrite/sender/response_test.go +++ b/remotewrite/sender/response_test.go @@ -23,7 +23,9 @@ import ( ) // TestResponseProcessing validates sender response header processing. -func TestResponseProcessing(t *testing.T) { +func TestResponseProcessing_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []struct { name string description string @@ -252,7 +254,9 @@ sample_3 3 } // TestContentTypeNegotiation validates content-type handling. -func TestContentTypeNegotiation(t *testing.T) { +func TestContentTypeNegotiation_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "SHOULD") t.Attr("description", "Sender SHOULD handle content-type negotiation") diff --git a/remotewrite/sender/retry_test.go b/remotewrite/sender/retry_test.go index 3e11d2bd..62ad04d1 100644 --- a/remotewrite/sender/retry_test.go +++ b/remotewrite/sender/retry_test.go @@ -23,7 +23,9 @@ import ( ) // TestRetryBehavior validates sender retry behavior on different error responses. -func TestRetryBehavior(t *testing.T) { +func TestRetryBehavior_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []struct { name string description string diff --git a/remotewrite/sender/rw1_compat_test.go b/remotewrite/sender/rw1_compat_test.go index e454cec2..0822332c 100644 --- a/remotewrite/sender/rw1_compat_test.go +++ b/remotewrite/sender/rw1_compat_test.go @@ -23,7 +23,9 @@ import ( // TestRemoteWrite1Compatibility validates RW 1.0 backward compatibility. // Note: These tests require sender to be configured for RW 1.0 mode. // Most senders default to RW 2.0, so RW 1.0 tests are informational. -func TestRemoteWrite1Compatibility(t *testing.T) { +func TestRemoteWrite1Compatibility_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "rw1_version_header", @@ -210,7 +212,9 @@ http_requests{method="POST",status="200"} 50 } // TestRemoteWrite1Configuration tests if sender can be configured for RW 1.0. -func TestRemoteWrite1Configuration(t *testing.T) { +func TestRemoteWrite1Configuration_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "SHOULD") t.Attr("description", "Sender SHOULD support RW 1.0 configuration for backward compatibility") diff --git a/remotewrite/sender/samples_test.go b/remotewrite/sender/samples_test.go index 60bb7b87..72bf885d 100644 --- a/remotewrite/sender/samples_test.go +++ b/remotewrite/sender/samples_test.go @@ -23,7 +23,9 @@ import ( ) // TestSampleEncoding validates that senders correctly encode float samples. -func TestSampleEncoding(t *testing.T) { +func TestSampleEncoding_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "float_value_encoding", @@ -224,7 +226,9 @@ func TestSampleEncoding(t *testing.T) { } // TestSampleOrdering validates timestamp ordering in samples. -func TestSampleOrdering(t *testing.T) { +func TestSampleOrdering_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "MUST") t.Attr("description", "Sender MUST send samples with older timestamps before newer ones within a series") diff --git a/remotewrite/sender/symbols_test.go b/remotewrite/sender/symbols_test.go index d4c4cd8e..99881559 100644 --- a/remotewrite/sender/symbols_test.go +++ b/remotewrite/sender/symbols_test.go @@ -21,7 +21,9 @@ import ( ) // TestSymbolTable validates symbol table requirements for Remote Write 2.0. -func TestSymbolTable(t *testing.T) { +func TestSymbolTable_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "empty_string_at_index_zero", @@ -105,7 +107,9 @@ another_metric{foo="bar"} 3 } // TestSymbolTableEfficiency validates that symbol tables are efficiently constructed. -func TestSymbolTableEfficiency(t *testing.T) { +func TestSymbolTableEfficiency_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + t.Attr("rfcLevel", "RECOMMENDED") t.Attr("description", "Symbol table should be efficiently constructed with good compression") diff --git a/remotewrite/sender/timestamps_test.go b/remotewrite/sender/timestamps_test.go index 813e9782..76411775 100644 --- a/remotewrite/sender/timestamps_test.go +++ b/remotewrite/sender/timestamps_test.go @@ -19,7 +19,9 @@ import ( ) // TestTimestampEncoding validates timestamp encoding and handling. -func TestTimestampEncoding(t *testing.T) { +func TestTimestampEncoding_Old(t *testing.T) { + t.Skip("TODO: Revise and move to a new framework") + tests := []TestCase{ { Name: "timestamp_int64_milliseconds", From c8512c2b08f960e19c5200aba3e777c3e4f76815 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 19 Feb 2026 10:08:49 +0000 Subject: [PATCH 02/13] rw2sender-fix: Add Makefile; clean README and CI (#210) * rw2sender-fix: Add Makefile; clean README and CI Signed-off-by: bwplotka * Apply suggestions from code review Signed-off-by: Bartlomiej Plotka * addressed comments Signed-off-by: bwplotka --------- Signed-off-by: bwplotka Signed-off-by: Bartlomiej Plotka --- .github/workflows/compliance-tests.yml | 30 +-- remotewrite/.gitignore | 4 + remotewrite/Makefile | 57 ++++++ remotewrite/README.md | 165 +++++++++------- remotewrite/index.html | 261 +++++++++++++++++++------ remotewrite/receiver/.gitignore | 2 - remotewrite/sender/main_test.go | 3 +- 7 files changed, 371 insertions(+), 151 deletions(-) create mode 100644 remotewrite/.gitignore create mode 100644 remotewrite/Makefile delete mode 100644 remotewrite/receiver/.gitignore diff --git a/.github/workflows/compliance-tests.yml b/.github/workflows/compliance-tests.yml index f0aa74fe..a24a8307 100644 --- a/.github/workflows/compliance-tests.yml +++ b/.github/workflows/compliance-tests.yml @@ -1,21 +1,21 @@ -name: Compliance Tests - +--- +name: Compliance on: pull_request: + push: + branches: [main] + +permissions: + contents: read jobs: - remotewrite-compliance: + remotewrite-sender: + name: Remote Write Sender runs-on: ubuntu-latest - + container: + image: quay.io/prometheus/golang-builder:1.25-base steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.22' - - - name: Run Remote Write Prometheus Compliance Tests - run: | - cd remotewrite/sender - go test --tags=compliance -run "TestRemoteWrite/prometheus/.+" -v ./ \ No newline at end of file + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - run: cd remotewrite && make sender \ No newline at end of file diff --git a/remotewrite/.gitignore b/remotewrite/.gitignore new file mode 100644 index 00000000..ab96d8cf --- /dev/null +++ b/remotewrite/.gitignore @@ -0,0 +1,4 @@ +results.json +config.yml +index.preloaded.html +bin/ \ No newline at end of file diff --git a/remotewrite/Makefile b/remotewrite/Makefile new file mode 100644 index 00000000..3066664c --- /dev/null +++ b/remotewrite/Makefile @@ -0,0 +1,57 @@ +# Set the default target to 'help' so it runs when you just type 'make' +.DEFAULT_GOAL := help + +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path)))) +ROOT_DIR := $(dir $(mkfile_path)) + +SED=$(shell which gsed 2>/dev/null || which sed) + +# Shared variables +DEBUG="" +TIMEOUT="10m" +PROMETHEUS_RW_COMPLIANCE_TEST_RE="" +PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE="" +RESULT_FILE="$(ROOT_DIR)/results.json" + +# Go test arguments and the base execution command. +BASE_TEST_ARGS = -run $(PROMETHEUS_RW_COMPLIANCE_TEST_RE) -skip $(PROMETHEUS_RW_COMPLIANCE_SKIP_TEST_RE) +RUN_TEST = cd ./$(COMPONENT) && DEBUG=$(DEBUG) $(COMPONENT_ENV) go test $(BASE_TEST_ARGS) + +# Use target-specific variables to dynamically inject directories and environment variables. +sender sender-%: PROMETHEUS_COMPLIANCE_RW_SENDERS="prometheus" +sender sender-%: COMPONENT = sender +sender sender-%: COMPONENT_ENV = PROMETHEUS_COMPLIANCE_RW_SENDERS=$(PROMETHEUS_COMPLIANCE_RW_SENDERS) + +receiver receiver-%: COMPONENT = receiver +receiver receiver-%: COMPONENT_ENV = PROMETHEUS_RW2_COMPLIANCE_RECEIVERS=$(PROMETHEUS_RW2_COMPLIANCE_RECEIVERS) + +.PHONY: help +help: ## Show this help message + @echo "Usage: make [target]" + @echo "" + @echo "Available commands:" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +.PHONY: sender +sender: ## Run sender compliance tests for a given senders suitable for humans and CI. + $(RUN_TEST) -v -timeout=$(TIMEOUT) + +.PHONY: sender-web +sender-web: ## Run sender compliance tests for a given senders with a HTML results view. + @$(RUN_TEST) -json | tee $(RESULT_FILE) + @$(MAKE) results-web + +.PHONY: receiver +receiver: ## Run receiver compliance tests for a given receivers suitable for humans and CI. + $(RUN_TEST) -v -timeout=$(TIMEOUT) + +.PHONY: receiver-web +receiver-web: ## Run receiver compliance tests for a given receivers with a HTML results view. + @$(RUN_TEST) -json | tee $(RESULT_FILE) + @$(MAKE) results-web + +.PHONY: results-web +results-web: ## Generate HTML view with ./results.json from the index.html file and open in your browser. We re-generate HTML file to avoid CORS issue on secure browsers. + @cat index.html | perl -pe 'BEGIN{local $$/; open(F,"; close F} s/const PRELOADED_RESULT_DATA = ``;/const PRELOADED_RESULT_DATA = `$$c`;/' > index.preloaded.html + @open index.preloaded.html \ No newline at end of file diff --git a/remotewrite/README.md b/remotewrite/README.md index 6fd589b2..59d16324 100644 --- a/remotewrite/README.md +++ b/remotewrite/README.md @@ -2,24 +2,90 @@ This repository contains compliance test suites for both **senders** and **receivers** of the Prometheus Remote-Write Protocol 2.0. It validates implementation of the Remote-Write v2 specification according to the [official protocol requirements](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/) as well as some more strict Prometheus implementation aspects. -## Overview +Both [sender](#sender-compliance-sender) and [receiver](#receiver-compliance-receiver) test suites uses a shared [`./index.html`](./index.html) file that enables viewing compliance test results after they complete. -The test suites validate protocol compliance for both sides of Remote-Write communication: +To generate and view results, add `-json` flag to `go test` command and `tee` the results to a file called `results.json`. -### Sender Compliance (`/sender`) -Tests that Remote-Write senders properly implement the protocol by forking sender instances (e.g., Prometheus), examining generated requests, and validating them against the specification. +Tests are marked with compliance levels according to RFC specifications: +- **MUST**: Required by Remote Write specification +- **SHOULD**: Recommended by Remote Write specification +- **MAY**: Could have by Remote Write specification +- **RECOMMENDED**: Not in Remote Write specification, but recommended for performance or Prometheus compatibility reasons + +## Sender Compliance (`/sender`) + +Tests that Remote-Write senders implement the RW2 protocol by running tests cases with: + +1. Scraper exposing OpenMetrics 1.0 +2. Target sender to test (e.g., Prometheus), +3. Special receiver that tests various elements of the request(s). Tests cover: +- Series encoding - Float samples encoding - Native Histograms encoding - Exemplars encoding -- Protocol headers and content negotiation -- Error handling and retry logic -- Backoff and batching behavior -- Metadata and symbol table management +- Metadata encoding +- Protocol headers +- Error handling, basic batching and retry logic - Request formatting and compression +- Various Prometheus / OpenMetrics 1.0 semantics as "RECOMMENDED" (staleness, upness). Because this test depends on scraping behaviour we +are testing some elements of the exposition format support too. + +### Prerequisites + +- Go 1.23 or later +- The sender binary you want to test (e.g., Prometheus) + - Requires OpenMetrics 1.0 scrape capability. + - Required Remote Write sending capability. + +The test suite automatically downloads and runs Prometheus as the reference sender implementation. + +### Configuration + +The test suite uses environment variables: + +**Sender Selection:** + +```bash +PROMETHEUS_COMPLIANCE_RW_SENDERS="prometheus" +``` + +Currently supported senders: +- `prometheus` - The reference Prometheus implementation (automatically downloaded). +- `process` - For custom sender that is a local binary. + +For testing custom senders: + +* Add target running code and register it the `sender/main_test.go`. + +TODO: Implement that. +* Use custom process target via `PROMETHEUS_COMPLIANCE_RW_SENDERS="process"` and `PROMETHEUS_COMPLIANCE_RW_PROCESS_BINARY=` envvars. + +**Debug output:** + +Debug variable controls if the tested process suppresses output (empty DEBUG) or not. + +```bash +DEBUG="1" +``` + +### Running Tests + +```bash +make sender +``` + +To use visualization HTML page: + +```bash +make sender-web +``` + +See Makefile for detailed invocation. + +## Receiver Compliance (`/receiver`) -### Receiver Compliance (`/receiver`) Tests that Remote-Write endpoints properly handle incoming requests by sending various Remote-Write v2 requests and validating responses. Tests cover: @@ -31,51 +97,24 @@ Tests cover: - Content-Type validation - Response headers (`X-Prometheus-Remote-Write-*-Written`) -## Limitations - -### Sender Tests -The test suite validates the format and structure of requests sent by the sender but does not verify end-to-end data flow or persistence. Because senders may have different configuration options and capabilities, passing all tests does not guarantee a sender supports every Remote-Write feature (such as Native Histograms). Some senders may not expose certain features or may require specific configuration. +### Limitations -### Receiver Tests This test suite does not verify data ingestion by reading data back from the receiver. Some requests that are valid for one backend might be rejected by another. The suite tolerates both 200 and 400 series HTTP responses since actual data validation is up to the receiver. Therefore, passing all tests does not guarantee that a receiver supports every Remote-Write feature. **Important**: You should review the detailed test output to judge compliance for your implementation. A successful `go test` run alone is not sufficient. -## Prerequisites - -### For Sender Tests -- Go 1.23 or later -- The sender binary to test (e.g., Prometheus) - -The test suite automatically downloads and runs Prometheus as the reference sender implementation. For testing custom senders, place the binary in the `bin/` directory. - -### For Receiver Tests -A Prometheus server with Remote-Write Receiver enabled, as baseline: -```bash -prometheus --web.enable-remote-write-receiver --enable-feature=native-histograms -``` - -Or any other Remote-Write Receiver endpoint. +### Prerequisites -## Configuration +You need a receiving endpoint you want to test. -### Sender Configuration -The test suite uses environment variables: +For example, a Prometheus server with Remote-Write Receiver enabled could be used as a baseline: -**Sender Selection:** ```bash -export PROMETHEUS_RW2_COMPLIANCE_SENDERS="prometheus" +prometheus --web.enable-remote-write-receiver --enable-feature=native-histograms # Add config file. ``` -Currently supported senders: -- `prometheus` - The reference Prometheus implementation (automatically downloaded) +### Configuration -**Test Timeout:** -```bash -export PROMETHEUS_RW2_COMPLIANCE_TEST_TIMEOUT="10m" -``` - -### Receiver Configuration The main configuration file `config.yml` in the `/remotewrite/receiver/` directory controls which receiver endpoints to test. It follows the Prometheus [`remote_write`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) structure: ```yaml @@ -94,48 +133,24 @@ If no `config.yml` exists, the test suite will fall back to `config_example.yml` Alternatively, use the `PROMETHEUS_RW2_COMPLIANCE_CONFIG_FILE` environment variable. **Receiver Filtering:** -```bash -export PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus,mimir" -``` -## Running Tests +You can configure which remote write endpoints to test from the provided `PROMETHEUS_RW2_COMPLIANCE_CONFIG_FILE` file +via the `PROMETHEUS_RW2_COMPLIANCE_RECEIVERS` environment variable: -### Sender Tests ```bash -PROMETHEUS_RW2_COMPLIANCE_SENDERS="prometheus" go test -v -# Or other sender if you want -``` - -### Receiver Tests -```bash -PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus" go test -v -# Or other receiver if you want +export PROMETHEUS_RW2_COMPLIANCE_RECEIVERS="local-prometheus,mimir" ``` -## HTML Visualization - -Both sender and receiver test suites include an `index.html` file that enables viewing compliance test results. - -To generate and view results: +### Running Tests -**For Sender Tests:** ```bash -cd sender -go test -json | tee results.json -# Open index.html in your browser +make receiver ``` -**For Receiver Tests:** +To use visualization HTML page: + ```bash -cd receiver -go test -json | tee results.json -# Open index.html in your browser +make receiver-web ``` -## Protocol Compliance Levels - -Tests are marked with compliance levels according to RFC specifications: -- **MUST**: Required by specification -- **SHOULD**: Recommended by specification -- **MAY**: Could have by specification -- **RECOMMENDED**: Not in specification but recommended for performance +See Makefile for detailed invocation. diff --git a/remotewrite/index.html b/remotewrite/index.html index 64a0063b..675844c0 100644 --- a/remotewrite/index.html +++ b/remotewrite/index.html @@ -1,3 +1,18 @@ + @@ -35,11 +50,6 @@

Remote-Write v2 — Test Matrix

- - Receiver - Sender - -