From 94fa6641ba059d5ed448aaf7486729f0bc71039f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Mon, 29 Jun 2026 13:41:28 -0400 Subject: [PATCH 01/77] feat: add NATS JetStream provider connector Add a natsjs connector implementing the provider.Provider contract so a NATS JetStream server can be used as an Arke backend, selectable per connection via ConnectionConfiguration.provider = "natsjs" alongside the existing amqp091 connector. The connector maps AMQP/RabbitMQ idioms onto JetStream primitives: - topic exchange + routing key -> subject root + token wildcards (# -> >, * -> *), with one stream per address root and per-source subject filters; - durable consumers for non-transient QUEUE / consumer-group / single-active sources (reconnect resumes the backlog) and ephemeral consumers for auto-delete / exclusive / temporary sources; - ack/nack/requeue/dead-letter onto Ack/Nak/NakWithDelay/Term, synthesizing the x-retry-count header from JetStream's delivery count; - publish dedup via Nats-Msg-Id + the stream Duplicates window; - header-filter exchanges evaluated proxy-side; - stream MaxAge/MaxBytes bounds (configurable) to keep the JetStream log from growing without limit, and a configurable replication factor for HA. Connection state is tracked via the NATS connection lifecycle callbacks so WaitForConnect reflects the real link. Behavioral tests run against an in-process JetStream server (publish/subscribe/ack/retry/dead-letter/dedup/ header-filter/durable-backlog); helper unit tests cover the subject mapping, durable-name selection, and filter evaluation. Signed-off-by: Michael Otteni --- go.mod | 9 + go.sum | 19 + internal/provider/connectors/connectors.go | 1 + .../provider/connectors/natsjs/helpers.go | 221 +++++++ .../connectors/natsjs/helpers_test.go | 133 ++++ internal/provider/connectors/natsjs/natsjs.go | 608 ++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 418 ++++++++++++ 7 files changed, 1409 insertions(+) create mode 100644 internal/provider/connectors/natsjs/helpers.go create mode 100644 internal/provider/connectors/natsjs/helpers_test.go create mode 100644 internal/provider/connectors/natsjs/natsjs.go create mode 100644 internal/provider/connectors/natsjs/natsjs_test.go diff --git a/go.mod b/go.mod index 7d8fde7..733a7bf 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/hashicorp/go-metrics v0.5.4 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.52.0 github.com/prometheus/client_golang v1.23.2 github.com/rabbitmq/amqp091-go v1.11.0 github.com/rabbitmq/rabbitmq-stream-go-client v1.8.1 @@ -29,6 +31,7 @@ require ( ) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -42,6 +45,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -51,9 +55,13 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/highwayhash v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect @@ -71,6 +79,7 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.45.0 // indirect diff --git a/go.sum b/go.sum index 19d426d..f984c2f 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -78,6 +80,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8 h1:ZI8gCoCjGzPsum4L21jHdQs8shFBIQih1TM9Rd/c+EQ= github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= @@ -134,6 +138,8 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -146,6 +152,16 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= @@ -254,6 +270,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= @@ -295,6 +313,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= diff --git a/internal/provider/connectors/connectors.go b/internal/provider/connectors/connectors.go index c5cebc0..0751663 100644 --- a/internal/provider/connectors/connectors.go +++ b/internal/provider/connectors/connectors.go @@ -5,4 +5,5 @@ package connectors import ( _ "github.com/sassoftware/arke/internal/provider/connectors/amqp091" // Import the AMQP091 plugin + _ "github.com/sassoftware/arke/internal/provider/connectors/natsjs" // Import the NATS JetStream plugin ) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go new file mode 100644 index 0000000..2e785c8 --- /dev/null +++ b/internal/provider/connectors/natsjs/helpers.go @@ -0,0 +1,221 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "strings" + + "github.com/nats-io/nats.go" + pb "github.com/sassoftware/arke/api" +) + +// Subject / topology mapping. +// +// AMQP (RabbitMQ) routes via exchange + routing-key bindings. NATS routes via +// subjects with token wildcards. We map: +// +// exchange/address name -> subject root (dots kept; it is just a multi-token prefix) +// routing key -> appended subject tokens +// AMQP '#' (zero+ words) -> NATS '>' (NATS only allows '>' as the final token) +// AMQP '*' (one word) -> NATS '*' +// +// A JetStream stream captures ".>" and consumers filter on the mapped +// source subjects. This is a faithful mapping for the dotted, topic-style keys +// AMQP topic exchanges use (e.g. orders.region.us.created.*). It does NOT +// reproduce AMQP headers-exchange routing — that is handled proxy-side in +// evaluateFilters (see Subscribe). + +// streamNameFor derives a JetStream-legal stream name from an address name. +// Stream names may not contain '.', ' ', '*' or '>'. +func streamNameFor(addressName string) string { + r := strings.NewReplacer(".", "_", " ", "_", "*", "_", ">", "_") + return "arke_" + r.Replace(addressName) +} + +// subjectFor joins an address root with a routing key / pattern, translating +// AMQP wildcards to NATS wildcards. +func subjectFor(addressName, routingKey string) string { + root := addressName + if routingKey == "" { + if root == "" { + return ">" + } + return root + ".>" + } + rk := translateWildcards(routingKey) + if rk == "" { + // The routing key was made up entirely of empty tokens (e.g. ".", + // trailing/leading dots): treat it as capture-all under the root. + if root == "" { + return ">" + } + return root + ".>" + } + if root == "" { + return rk + } + return root + "." + rk +} + +// tokenSanitizer replaces characters that are illegal inside a NATS subject +// token (space/tab, and the wildcard chars '*'/'>') so a literal AMQP token that +// happens to embed one of them still yields a valid subject. +var tokenSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_") + +// translateWildcards converts an AMQP topic routing key into a valid NATS subject +// fragment. NATS is stricter than AMQP about subjects, so besides translating the +// wildcards it also sanitizes each token. Mapping (per dot-delimited token): +// +// "*" -> "*" single-token wildcard (same meaning) +// "#" (last token) -> ">" tail wildcard (same meaning) +// "#" (mid token) -> "*" NATS '>' is tail-only, so a non-terminal AMQP '#' +// has no exact equivalent; '*' keeps the subject valid +// but narrows zero-or-more to exactly-one. +// "" -> dropped (collapses "a..b" and leading/trailing dots, which +// would otherwise be empty — and thus invalid — tokens) +// other -> illegal chars (space/tab/'*'/'>') replaced with '_' +// +// The result is always a valid NATS subject fragment (possibly empty if every +// token was empty; subjectFor handles that). +func translateWildcards(key string) string { + tokens := strings.Split(key, ".") + out := make([]string, 0, len(tokens)) + for i, t := range tokens { + switch t { + case "*": + out = append(out, "*") + case "#": + if i == len(tokens)-1 { + out = append(out, ">") + } else { + out = append(out, "*") + } + case "": + // drop empty token + default: + out = append(out, tokenSanitizer.Replace(t)) + } + } + return strings.Join(out, ".") +} + +// streamSubjectsFor returns the wildcard subject set a stream should capture for +// the given address. +func streamSubjectsFor(addressName string) []string { + if addressName == "" { + return []string{">"} + } + return []string{addressName + ".>"} +} + +// filterSubjectsFor maps a source's subjects to NATS consumer filter subjects. +func filterSubjectsFor(source *pb.Source) []string { + addr := source.GetAddress().GetName() + subs := source.GetAddress().GetSubjects() + if len(subs) == 0 { + return []string{subjectFor(addr, "")} + } + out := make([]string, 0, len(subs)) + for _, s := range subs { + out = append(out, subjectFor(addr, s)) + } + return out +} + +// durableName returns the JetStream durable consumer name for a source, or "" +// if the source should use an ephemeral consumer. +// +// Durable (work-queue) consumers persist across client reconnects, so a backlog +// published while the consumer is disconnected is redelivered on reconnect — +// matching RabbitMQ durable-queue semantics. Transient sources stay ephemeral +// and auto-expire (InactiveThreshold). +// +// Clients commonly mark per-instance/transient consumers by converting an +// Exclusive queue into AutoDelete + a UUID-suffixed name, so +// auto-delete/exclusive/TEMPORARY sources are treated as the transient ones. A +// non-transient QUEUE (a durable named listener) gets a durable consumer, as does +// a STREAM source with a ConsumerGroup, or any SingleActiveConsumer. +func durableName(source *pb.Source) string { + if source.GetAutoDelete() || source.GetExclusive() || source.GetType() == pb.Source_TEMPORARY { + return "" + } + if source.GetSingleActiveConsumer() { + return streamNameFor(source.GetName()) + } + switch source.GetType() { + case pb.Source_QUEUE: + return streamNameFor(source.GetName()) + case pb.Source_STREAM: + if cg := source.GetOptions()["ConsumerGroup"]; cg != "" { + return streamNameFor(cg) + } + case pb.Source_TEMPORARY: + // Transient; already returned above via the TEMPORARY guard. + } + return "" +} + +// pbToNatsHeader converts Arke's flat string headers to a NATS header. +func pbToNatsHeader(in map[string]string) nats.Header { + if len(in) == 0 { + return nil + } + h := nats.Header{} + for k, v := range in { + h.Set(k, v) + } + return h +} + +// natsToPbHeader converts a NATS header to Arke's flat string headers, keeping +// the first value of each key. +func natsToPbHeader(in nats.Header) map[string]string { + if len(in) == 0 { + return map[string]string{} + } + out := make(map[string]string, len(in)) + for k, vals := range in { + if len(vals) > 0 { + out[k] = vals[0] + } + } + return out +} + +// evaluateFilters reproduces RabbitMQ headers-exchange matching proxy-side. +// +// Multiple Filters are OR'd together (each corresponds to a separate AMQP +// binding). Within a single Filter, matches are combined per Filter.Type +// (ALL = and, ANY = or). No filters means the message always passes. +func evaluateFilters(filters []*pb.Filter, headers map[string]string) bool { + if len(filters) == 0 { + return true + } + for _, f := range filters { + if filterMatches(f, headers) { + return true + } + } + return false +} + +func filterMatches(f *pb.Filter, headers map[string]string) bool { + matches := f.GetMatches() + if len(matches) == 0 { + return true + } + any := f.GetType() == pb.Filter_ANY + for _, m := range matches { + got, ok := headers[m.GetName()] + hit := ok && got == m.GetValue() + if any && hit { + return true + } + if !any && !hit { + return false + } + } + // ALL: every match passed; ANY: none passed. + return !any +} diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go new file mode 100644 index 0000000..fdc98a2 --- /dev/null +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -0,0 +1,133 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "testing" + + pb "github.com/sassoftware/arke/api" + "github.com/stretchr/testify/assert" +) + +func TestStreamNameFor(t *testing.T) { + assert.Equal(t, "arke_events_orders", streamNameFor("events.orders")) + assert.Equal(t, "arke_events_audit", streamNameFor("events.audit")) + // illegal subject chars are sanitized + assert.Equal(t, "arke_a_b_c", streamNameFor("a.b*c")) +} + +func TestSubjectFor(t *testing.T) { + // dotted topic key under an exchange root + assert.Equal(t, + "events.orders.region.us.order.created.success", + subjectFor("events.orders", "region.us.order.created.success")) + // AMQP '#' -> NATS '>' + assert.Equal(t, "events.orders.region.>", subjectFor("events.orders", "region.#")) + // AMQP '*' stays '*' + assert.Equal(t, "events.orders.region.*.success", subjectFor("events.orders", "region.*.success")) + // empty routing key -> capture-all under root + assert.Equal(t, "events.orders.>", subjectFor("events.orders", "")) +} + +// TestSubjectForSanitizes covers routing keys that are legal in AMQP but would +// otherwise yield an invalid NATS subject. +func TestSubjectForSanitizes(t *testing.T) { + // non-terminal '#' has no exact NATS equivalent ('>' is tail-only) -> '*' + assert.Equal(t, "events.orders.a.*.b", subjectFor("events.orders", "a.#.b")) + // terminal '#' still maps to '>' + assert.Equal(t, "events.orders.a.b.>", subjectFor("events.orders", "a.b.#")) + // empty tokens (double dot / trailing dot) are collapsed, not left invalid + assert.Equal(t, "events.orders.a.b", subjectFor("events.orders", "a..b")) + assert.Equal(t, "events.orders.a.b", subjectFor("events.orders", "a.b.")) + // a routing key of only empty tokens -> capture-all under root + assert.Equal(t, "events.orders.>", subjectFor("events.orders", ".")) + // illegal chars embedded in a literal token are replaced with '_' + assert.Equal(t, "events.orders.a_b.c_d", subjectFor("events.orders", "a*b.c>d")) + assert.Equal(t, "events.orders.a_b", subjectFor("events.orders", "a b")) +} + +func TestStreamSubjectsFor(t *testing.T) { + assert.Equal(t, []string{"events.orders.>"}, streamSubjectsFor("events.orders")) + assert.Equal(t, []string{">"}, streamSubjectsFor("")) +} + +func TestDurableName(t *testing.T) { + q := func(name string) *pb.Source { + return &pb.Source{Name: name, Address: &pb.Address{Name: "events.orders"}} + } + + // non-transient QUEUE (default type, not auto-delete) -> durable + assert.Equal(t, "arke_events_orders_shipping_listener", + durableName(q("events.orders.shipping.listener"))) + + // auto-delete (transient/exclusive conversion) -> ephemeral + ad := q("sess-abc") + ad.AutoDelete = true + assert.Equal(t, "", durableName(ad)) + + // exclusive -> ephemeral + ex := q("sess-xyz") + ex.Exclusive = true + assert.Equal(t, "", durableName(ex)) + + // TEMPORARY type -> ephemeral + tmp := q("tmp") + tmp.Type = pb.Source_TEMPORARY + assert.Equal(t, "", durableName(tmp)) + + // SingleActiveConsumer -> durable even if otherwise plain + sac := q("sac") + sac.SingleActiveConsumer = true + assert.Equal(t, "arke_sac", durableName(sac)) + + // STREAM with ConsumerGroup -> durable (named after the group) + st := q("st") + st.Type = pb.Source_STREAM + st.Options = map[string]string{"ConsumerGroup": "grp.a"} + assert.Equal(t, "arke_grp_a", durableName(st)) + + // STREAM without ConsumerGroup -> ephemeral + st2 := q("st2") + st2.Type = pb.Source_STREAM + assert.Equal(t, "", durableName(st2)) +} + +func TestEvaluateFilters(t *testing.T) { + headers := map[string]string{"event-source": "orders", "region": "us"} + + // no filters -> always pass + assert.True(t, evaluateFilters(nil, headers)) + + // ALL: every match must hit + all := &pb.Filter{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{ + {Name: "event-source", Value: "orders"}, + {Name: "region", Value: "us"}, + }, + } + assert.True(t, evaluateFilters([]*pb.Filter{all}, headers)) + + allMiss := &pb.Filter{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{ + {Name: "event-source", Value: "orders"}, + {Name: "region", Value: "eu"}, + }, + } + assert.False(t, evaluateFilters([]*pb.Filter{allMiss}, headers)) + + // ANY: a single hit passes + any := &pb.Filter{ + Type: pb.Filter_ANY, + Matches: []*pb.Match{ + {Name: "event-source", Value: "nope"}, + {Name: "region", Value: "us"}, + }, + } + assert.True(t, evaluateFilters([]*pb.Filter{any}, headers)) + + // multiple filters are OR'd: second filter matches + assert.True(t, evaluateFilters([]*pb.Filter{allMiss, all}, headers)) +} diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go new file mode 100644 index 0000000..c7f2a57 --- /dev/null +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -0,0 +1,608 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package natsjs is a NATS JetStream backend for Arke. It implements the +// provider.Provider contract so it can be selected via +// ConnectionConfiguration.provider = "natsjs", side by side with amqp091. +// +// It covers the core publish / subscribe / ack / delayed-retry / dead-letter / +// dedup / header-filter paths an AMQP client exercises through Arke, mapping +// each onto a native JetStream primitive where one exists and onto a small +// amount of proxy-side translation where it does not. It is not a one-to-one +// replacement for every RabbitMQ feature — see doc/design/natsjs-connector.md +// for the per-feature parity matrix and known limitations. +package natsjs + +import ( + "context" + "crypto/tls" + "fmt" + "os" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/internal/provider" + "github.com/sassoftware/arke/internal/util" +) + +const providerName = "natsjs" + +// retryCountHeaderName mirrors the amqp091 connector. Clients that count retries +// via this header (rather than RabbitMQ's x-death) keep working unchanged because +// the connector synthesizes it from JetStream's delivery count — JetStream has no +// x-death equivalent. +const retryCountHeaderName = "x-retry-count" + +const ( + defaultAckWait = 30 * time.Second + defaultInactiveThreshold = 5 * time.Minute + defaultDedupWindow = 2 * time.Minute + connectPollInterval = 50 * time.Millisecond + // defaultStreamMaxAge bounds how long a stream retains messages so the + // JetStream log does not grow without bound (RabbitMQ deletes on ack; + // JetStream LimitsPolicy does not). 72h is generous enough not to truncate a + // realistic consumer-outage backlog, short enough to stop indefinite + // accumulation. Override via NATSJS_STREAM_MAX_AGE. + defaultStreamMaxAge = 72 * time.Hour +) + +// supportedSourceOptionsList intentionally matches the amqp091 connector so that +// existing client Sources validate against the server unchanged. +var supportedSourceOptionsList = []string{ + "MessageTTL", "DeadLetterAddress", "DeadLetterSubject", "Expires", "Offset", "ConsumerGroup", +} + +var supportedSourceOptions map[string]bool + +// GetClientIdentifier is a var so tests can override it (matches amqp091). +var GetClientIdentifier = util.GetClientIdentifier + +type natsBrokerDetails struct { + sync.Mutex + nc *nats.Conn + js jetstream.JetStream + clientIdentifier string + connectionConfig *pb.ConnectionConfiguration + + // activeMessages maps an Arke message UUID -> the in-flight jetstream.Msg + // so a later Ack/Nack/Retry/DeadLetter RPC can resolve it. + activeMessages *util.ConcurrentMap + // knownStreams memoizes streams we have already ensured. + knownStreams *util.ConcurrentMap + // consumeContexts maps source name -> jetstream.ConsumeContext for teardown. + consumeContexts *util.ConcurrentMap + + state atomic.Uint32 + consumed int64 + produced int64 +} + +type natsjsProvider struct { + connections *util.ConcurrentMap +} + +func init() { + provider.Register(providerName, NewNATSJetStreamProvider) + supportedSourceOptions = make(map[string]bool, len(supportedSourceOptionsList)) + for _, o := range supportedSourceOptionsList { + supportedSourceOptions[o] = true + } +} + +// NewNATSJetStreamProvider returns a new natsjs provider singleton. +func NewNATSJetStreamProvider() provider.Provider { + return &natsjsProvider{connections: util.NewConcurrentMap()} +} + +// streamReplicas controls the JetStream replication factor (1 = single node, +// 3 = R3/Raft for HA). Configured via NATSJS_STREAM_REPLICAS. +func streamReplicas() int { + if v := os.Getenv("NATSJS_STREAM_REPLICAS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 1 +} + +// streamMaxAge is the primary guard against unbounded JetStream log growth. +// Configured via NATSJS_STREAM_MAX_AGE as a Go duration ("72h", "24h", "0" = +// keep forever); defaults to defaultStreamMaxAge. +// +// Note: because the connector uses one stream per address root, this is a +// stream-wide bound, NOT a faithful per-source x-message-ttl. A per-source +// MessageTTL is deliberately NOT folded in here: ensureStream is also called from +// the publish path (which has no Source), so mixing a per-source TTL with the +// global default would make MaxAge flap on the shared stream depending on whether +// a publish or a subscribe touched it last. Per-source TTL fidelity needs a +// per-source stream topology — see the "Known limitations" section of +// doc/design/natsjs-connector.md. +func streamMaxAge() time.Duration { + if v := os.Getenv("NATSJS_STREAM_MAX_AGE"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d >= 0 { + return d + } + } + return defaultStreamMaxAge +} + +// streamMaxBytes optionally caps a stream's on-disk size as a hard storage +// safety net. With the default discard=old, JetStream evicts the oldest messages +// only when near the cap, so recent backlog is preserved as long as possible. +// Configured via NATSJS_STREAM_MAX_BYTES (bytes); default 0 = unlimited (rely on +// MaxAge). Set it to bound disk per stream when volume is high. +func streamMaxBytes() int64 { + if v := os.Getenv("NATSJS_STREAM_MAX_BYTES"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return 0 +} + +func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return nil, err + } + if bd := p.getBrokerDetailsByIdentifier(clientID); bd != nil { + return bd, nil + } + return nil, fmt.Errorf("could not retrieve broker details for connection: %s", clientID) +} + +func (p *natsjsProvider) getBrokerDetailsByIdentifier(id string) *natsBrokerDetails { + if bd, ok := p.connections.Get(id); ok { + return bd.(*natsBrokerDetails) + } + return nil +} + +func (p *natsjsProvider) ClientExists(id string) bool { + return p.getBrokerDetailsByIdentifier(id) != nil +} + +// Connect establishes a NATS connection and JetStream context for the client. +func (p *natsjsProvider) Connect(ctx context.Context, cfg *pb.ConnectionConfiguration, tlsSkipVerify bool) *pb.Error { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + + scheme := "nats://" + if cfg.GetTls() { + scheme = "tls://" + } + url := fmt.Sprintf("%s%s:%d", scheme, cfg.GetHost(), cfg.GetPort()) + + bd := &natsBrokerDetails{ + clientIdentifier: clientID, + connectionConfig: cfg, + activeMessages: util.NewConcurrentMap(), + knownStreams: util.NewConcurrentMap(), + consumeContexts: util.NewConcurrentMap(), + } + bd.state.Store(provider.CONNECTING) + + opts := []nats.Option{ + nats.Name(cfg.GetClientName()), + nats.MaxReconnects(-1), + nats.RetryOnFailedConnect(true), + // Keep bd.state honest so WaitForConnect reflects the real link. With + // RetryOnFailedConnect, nats.Connect returns a non-error connection even + // when the broker is unreachable; nats.go then reconnects in the + // background (MaxReconnects(-1)). These callbacks mirror the amqp091 + // connectionWatcher: WaitForConnect waits out a broker outage instead of + // falsely reporting CONNECTED. + nats.ConnectHandler(func(*nats.Conn) { bd.state.Store(provider.CONNECTED) }), + nats.ReconnectHandler(func(*nats.Conn) { bd.state.Store(provider.CONNECTED) }), + nats.DisconnectErrHandler(func(_ *nats.Conn, _ error) { bd.state.Store(provider.DISCONNECTED) }), + nats.ClosedHandler(func(*nats.Conn) { bd.state.Store(provider.CLOSED) }), + } + if creds := cfg.GetCredentials(); creds != nil && creds.GetUsername() != "" { + opts = append(opts, nats.UserInfo(creds.GetUsername(), creds.GetPassword())) + } + if cfg.GetTls() { + // The broker certificate is verified against the system trust store; + // client-provided CA certificates are not supported (matches amqp091). + tlsCfg := &tls.Config{InsecureSkipVerify: tlsSkipVerify} //nolint:gosec // operator opt-in, mirrors amqp091 + opts = append(opts, nats.Secure(tlsCfg)) + } + + nc, err := nats.Connect(url, opts...) + if err != nil { + return &pb.Error{Message: fmt.Sprintf("nats connect failed: %s", err.Error()), IsFatal: true} + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return &pb.Error{Message: fmt.Sprintf("jetstream init failed: %s", err.Error()), IsFatal: true} + } + + bd.nc = nc + bd.js = js + // If the broker was reachable, nats.Connect already completed the handshake + // (the ConnectHandler may not fire for a synchronous connect), so reflect the + // live status here; otherwise stay CONNECTING until a callback flips it. + if nc.IsConnected() { + bd.state.Store(provider.CONNECTED) + } + p.connections.Add(clientID, bd) + util.Logger.Debugf("natsjs: client %s connecting to %s", clientID, url) + return nil +} + +func (p *natsjsProvider) WaitForConnect(ctx context.Context) bool { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return false + } + deadline := time.Now().Add(provider.CONNECTTIMEOUT * time.Second) + for time.Now().Before(deadline) { + if bd := p.getBrokerDetailsByIdentifier(clientID); bd != nil && bd.state.Load() == provider.CONNECTED { + return true + } + select { + case <-ctx.Done(): + return false + case <-time.After(connectPollInterval): + } + } + return false +} + +func (p *natsjsProvider) Disconnect(ctx context.Context) { + clientID, err := GetClientIdentifier(ctx) + if err != nil { + return + } + bd := p.getBrokerDetailsByIdentifier(clientID) + if bd == nil { + return + } + for _, name := range bd.consumeContexts.GetList() { + if cc, ok := bd.consumeContexts.Get(name); ok { + cc.(jetstream.ConsumeContext).Stop() + } + } + if bd.nc != nil { + _ = bd.nc.Drain() + } + bd.state.Store(provider.CLOSED) + p.connections.Delete(clientID) + util.Logger.Debugf("natsjs: client %s disconnected", clientID) +} + +// ensureStream lazily creates (or updates) a JetStream stream that captures all +// subjects under the given address root. +func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails, addressName string) (string, error) { + streamName := streamNameFor(addressName) + if _, ok := bd.knownStreams.Get(streamName); ok { + return streamName, nil + } + _, err := bd.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: streamSubjectsFor(addressName), + Storage: jetstream.FileStorage, + Retention: jetstream.LimitsPolicy, + Duplicates: defaultDedupWindow, + // Replicas via NATSJS_STREAM_REPLICAS (default 1). Set to 3 against a + // clustered nats-server for the HA-equivalent of RabbitMQ quorum queues. + Replicas: streamReplicas(), + // MaxAge/MaxBytes bound the log so it does not grow without limit the way + // the default LimitsPolicy otherwise would (RabbitMQ deletes on ack; + // JetStream retains acked messages). MaxAge is the time guard, MaxBytes an + // optional hard storage cap. Both are mutable, so this also reins in + // streams created before the limits existed. See natsjs-connector.md. + MaxAge: streamMaxAge(), + MaxBytes: streamMaxBytes(), + }) + if err != nil { + return "", err + } + bd.knownStreams.Add(streamName, true) + return streamName, nil +} + +func firstSubject(addr *pb.Address) string { + if subs := addr.GetSubjects(); len(subs) > 0 { + return subs[0] + } + return "" +} + +// Publish drains the message channel, publishing each message to JetStream. +func (p *natsjsProvider) Publish(ctx context.Context, in <-chan *pb.Message, errChan chan<- *pb.Error) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + for { + select { + case <-ctx.Done(): + return nil + case msg, ok := <-in: + if !ok || msg == nil { + return nil + } + // The server blocks on <-errChan for every message it hands the + // provider (server.go: `mc <- msg` then `pubErr := <-ec`), so we + // must always reply exactly once — nil on success. + errChan <- p.publishMsg(ctx, bd, msg) + } + } +} + +func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + return p.publishMsg(ctx, bd, msg) +} + +func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message) *pb.Error { + addr := msg.GetAddress() + if _, err := p.ensureStream(ctx, bd, addr.GetName()); err != nil { + return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} + } + nmsg := &nats.Msg{ + Subject: subjectFor(addr.GetName(), firstSubject(addr)), + Data: msg.GetBody(), + Header: pbToNatsHeader(msg.GetHeaders()), + } + var pubOpts []jetstream.PublishOpt + // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). + if msg.GetPublisherName() != "" || msg.GetPublishId() != 0 { + pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) + } + if _, err := bd.js.PublishMsg(ctx, nmsg, pubOpts...); err != nil { + return &pb.Error{Message: fmt.Sprintf("publish: %s", err.Error())} + } + atomic.AddInt64(&bd.produced, 1) + return nil +} + +// Subscribe ensures topology, creates a consumer, and forwards deliveries to the +// message channel until the context is cancelled. Blocks per the Provider +// contract. +func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out chan<- *pb.Message) *pb.Error { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error(), IsFatal: true} + } + addr := source.GetAddress().GetName() + streamName, serr := p.ensureStream(ctx, bd, addr) + if serr != nil { + return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", serr.Error()), IsFatal: true} + } + stream, serr := bd.js.Stream(ctx, streamName) + if serr != nil { + return &pb.Error{Message: fmt.Sprintf("open stream: %s", serr.Error()), IsFatal: true} + } + + prefetch := int(source.GetPrefetchCount()) + if prefetch <= 0 { + prefetch = 1 + } + consCfg := jetstream.ConsumerConfig{ + FilterSubjects: filterSubjectsFor(source), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: deliverPolicyFor(source), + AckWait: defaultAckWait, + MaxAckPending: prefetch, + } + // Durable work-queue consumer for non-transient QUEUE / stream-group sources: + // it persists across client reconnects so a backlog published during a + // consumer outage is redelivered (RabbitMQ durable-queue parity). The + // DeliverPolicy only applies on first creation; reconnects resume from the + // durable's stored ack position. Transient (auto-delete/exclusive/TEMPORARY) + // sources stay ephemeral and auto-expire after InactiveThreshold. + if durable := durableName(source); durable != "" { + consCfg.Durable = durable + } else { + consCfg.InactiveThreshold = defaultInactiveThreshold + } + + cons, cerr := stream.CreateOrUpdateConsumer(ctx, consCfg) + if cerr != nil { + // Include the mapped filter subjects: a NATS "invalid subject" (10052) + // otherwise gives no hint which source/subject produced it. + return &pb.Error{ + Message: fmt.Sprintf("create consumer (filters=%v): %s", consCfg.FilterSubjects, cerr.Error()), + IsFatal: true, + } + } + + // DeclareOnly: topology established, do not consume. + if source.GetDeclareOnly() { + return nil + } + + cc, cerr := cons.Consume(func(m jetstream.Msg) { + p.handleDelivery(ctx, bd, source, m, out) + }) + if cerr != nil { + return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} + } + bd.consumeContexts.Add(source.GetName(), cc) + defer func() { + cc.Stop() + bd.consumeContexts.Delete(source.GetName()) + }() + + <-ctx.Done() + return nil +} + +func deliverPolicyFor(source *pb.Source) jetstream.DeliverPolicy { + // Maps the Offset option. Transient consumers typically want "new" (only + // messages after subscribe); stream replay consumers want "all". + switch source.GetOptions()["Offset"] { + case "first": + return jetstream.DeliverAllPolicy + case "next", "": + return jetstream.DeliverNewPolicy + default: + return jetstream.DeliverNewPolicy + } +} + +func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, m jetstream.Msg, out chan<- *pb.Message) { + defer util.RecoverPanic() + + headers := natsToPbHeader(m.Headers()) + // Synthesize x-retry-count from JetStream's delivery count so a client's retry + // policy works without RabbitMQ's x-death header. + if md, err := m.Metadata(); err == nil && md.NumDelivered > 1 { + headers[retryCountHeaderName] = strconv.FormatUint(md.NumDelivered-1, 10) + } + + // Proxy-side replacement for RabbitMQ headers-exchange routing. + if !evaluateFilters(source.GetFilters(), headers) { + _ = m.Ack() + return + } + + uuid := util.GenUUID() + bd.activeMessages.Add(uuid, m) + pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: m.Data()} + + select { + case out <- pbmsg: + atomic.AddInt64(&bd.consumed, 1) + case <-ctx.Done(): + bd.activeMessages.Delete(uuid) + } +} + +func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstream.Msg, *natsBrokerDetails, *pb.Error) { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return nil, nil, &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return nil, bd, &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} + } + bd.activeMessages.Delete(uuid) + return mu.(jetstream.Msg), bd, nil +} + +func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { + m, _, perr := p.takeMessage(ctx, uuid) + if perr != nil { + return perr + } + if err := m.Ack(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +// Nack negatively acknowledges for immediate redelivery (literal AMQP nack). +func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { + m, _, perr := p.takeMessage(ctx, uuid) + if perr != nil { + return perr + } + if err := m.Nak(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +// Retry requeues the message after a delay. NakWithDelay is the native +// JetStream primitive that replaces RabbitMQ's per-message-TTL + dead-letter +// retry-queue idiom; JetStream increments the delivery count for us, which +// handleDelivery surfaces as x-retry-count. +func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, delay int32) *pb.Error { + m, _, perr := p.takeMessage(ctx, uuid) + if perr != nil { + return perr + } + if err := m.NakWithDelay(time.Duration(delay) * time.Second); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +// DeadLetter publishes the message to the configured dead-letter subject (there +// is no native DLX in JetStream) and terminates redelivery. +func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid string) *pb.Error { + m, bd, perr := p.takeMessage(ctx, uuid) + if perr != nil { + return perr + } + opts := source.GetOptions() + if dla := opts["DeadLetterAddress"]; dla != "" { + if _, err := p.ensureStream(ctx, bd, dla); err == nil { + dlqMsg := &nats.Msg{ + Subject: subjectFor(dla, opts["DeadLetterSubject"]), + Data: m.Data(), + Header: m.Headers(), + } + if _, err := bd.js.PublishMsg(ctx, dlqMsg); err != nil { + util.Logger.Debugf("natsjs: dead-letter publish failed for %s: %s", uuid, err.Error()) + } + } + } + if err := m.Term(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil +} + +func (p *natsjsProvider) SupportedSourceOptions() map[string]bool { + return supportedSourceOptions +} + +func (p *natsjsProvider) Stats() *provider.Stats { + stats := &provider.Stats{} + for _, id := range p.connections.GetList() { + bd := p.getBrokerDetailsByIdentifier(id) + if bd == nil { + continue + } + stats.Clients = append(stats.Clients, &provider.ClientStats{ + ID: id, + ActiveMessages: bd.activeMessages.Length(), + Streams: bd.consumeContexts.Length(), + Produced: int(atomic.LoadInt64(&bd.produced)), + Consumed: int(atomic.LoadInt64(&bd.consumed)), + }) + } + return stats +} + +func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb.SourceStats { + stats := &pb.SourceStats{Name: source.GetName()} + bd, err := p.getBrokerDetails(ctx) + if err != nil { + stats.Error = &pb.Error{Message: err.Error()} + return stats + } + streamName := streamNameFor(source.GetAddress().GetName()) + stream, serr := bd.js.Stream(ctx, streamName) + if serr != nil { + stats.Error = &pb.Error{Message: serr.Error()} + return stats + } + info, ierr := stream.Info(ctx) + if ierr != nil { + stats.Error = &pb.Error{Message: ierr.Error()} + return stats + } + stats.MessageCount = int64(info.State.Msgs) //nolint:gosec + stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec + stats.LastOffset = int64(info.State.LastSeq) //nolint:gosec + // Note: publish_rate / deliver_rate are not exposed directly by + // JetStream stream info; they would need to be derived from sampling. Left + // at zero for now. + return stats +} diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go new file mode 100644 index 0000000..03a4f42 --- /dev/null +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -0,0 +1,418 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package natsjs + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go/jetstream" + pb "github.com/sassoftware/arke/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The connector resolves the per-client broker state from the client identifier +// in the context. For tests we override it with a fixed identifier (the real +// implementation reads it from gRPC metadata). +func init() { + GetClientIdentifier = func(context.Context) (string, error) { + return "test-client", nil + } +} + +// runJetStreamServer starts an in-process NATS server with JetStream enabled, +// backed by a temp store dir that is cleaned up with the test. +func runJetStreamServer(t *testing.T) *natsserver.Server { + t.Helper() + opts := &natsserver.Options{ + Host: "127.0.0.1", + Port: -1, // choose a free port + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + s, err := natsserver.NewServer(opts) + require.NoError(t, err) + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("embedded nats server did not become ready") + } + t.Cleanup(s.Shutdown) + return s +} + +// connectClient builds a provider and connects a single client to the server. +func connectClient(t *testing.T, s *natsserver.Server) (*natsjsProvider, context.Context) { + t.Helper() + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + cfg := &pb.ConnectionConfiguration{ + Host: "127.0.0.1", + Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: "test", + } + ctx := context.Background() + if perr := p.Connect(ctx, cfg, false); perr != nil { + t.Fatalf("connect: %s", perr.GetMessage()) + } + if !p.WaitForConnect(ctx) { + t.Fatal("WaitForConnect returned false against a live server") + } + return p, ctx +} + +func queueSource(name, addr string, subjects ...string) *pb.Source { + return &pb.Source{ + Name: name, + Type: pb.Source_QUEUE, + Address: &pb.Address{Name: addr, Subjects: subjects}, + // Offset:first => deliver from the start so tests can publish before + // subscribing without racing the consumer setup. + Options: map[string]string{"Offset": "first"}, + } +} + +// recv waits for one message on out or fails after a generous timeout. +func recv(t *testing.T, out <-chan *pb.Message) *pb.Message { + t.Helper() + select { + case m := <-out: + return m + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for a message") + return nil + } +} + +func TestConnectDisconnect(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + assert.True(t, p.ClientExists("test-client")) + + p.Disconnect(ctx) + assert.False(t, p.ClientExists("test-client")) +} + +func TestSupportedSourceOptions(t *testing.T) { + p := NewNATSJetStreamProvider().(*natsjsProvider) + opts := p.SupportedSourceOptions() + for _, k := range []string{"MessageTTL", "DeadLetterAddress", "DeadLetterSubject", "Expires", "Offset", "ConsumerGroup"} { + assert.True(t, opts[k], "expected %s to be supported", k) + } + assert.False(t, opts["NotARealOption"]) +} + +func TestPublishOneAndSubscribe(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.orders", Subjects: []string{"created"}} + const n = 5 + for i := 0; i < n; i++ { + msg := &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))} + if perr := p.PublishOne(ctx, msg); perr != nil { + t.Fatalf("publish %d: %s", i, perr.GetMessage()) + } + } + + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.orders.consumer", "events.orders", "created"), out) + + for i := 0; i < n; i++ { + m := recv(t, out) + assert.NoError(t, errOf(p.Ack(ctx, m.GetUuid()))) + } + + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Equal(t, n, stats.Clients[0].Produced) + assert.Equal(t, n, stats.Clients[0].Consumed) +} + +func TestPublishChannel(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message) + errChan := make(chan *pb.Error) + go func() { _ = p.Publish(ctx, in, errChan) }() + + addr := &pb.Address{Name: "events.audit", Subjects: []string{"login"}} + // The server protocol sends a message then blocks reading exactly one error. + in <- &pb.Message{Address: addr, Body: []byte("hello")} + assert.Nil(t, <-errChan, "publish should report nil on success") + close(in) +} + +func TestRetrySynthesizesRetryCount(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.retry", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.retry.consumer", "events.retry", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + assert.Empty(t, first.GetHeaders()[retryCountHeaderName], "first delivery has no retry count") + require.Nil(t, p.Retry(ctx, src, first.GetUuid(), 1)) + + second := recv(t, out) + assert.Equal(t, "1", second.GetHeaders()[retryCountHeaderName], "redelivery carries x-retry-count") + require.Nil(t, p.Ack(ctx, second.GetUuid())) +} + +func TestNackRedelivers(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.nack", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.nack.consumer", "events.nack", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + require.Nil(t, p.Nack(ctx, first.GetUuid())) + + second := recv(t, out) + require.Nil(t, p.Ack(ctx, second.GetUuid())) +} + +func TestDeadLetter(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dl", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dl.consumer", "events.dl", "job") + src.Options["DeadLetterAddress"] = "events.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // The dead-lettered copy lands on the DLQ stream. + dlqStats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dlq.consumer", + Address: &pb.Address{Name: "events.dlq"}, + }) + assert.Nil(t, dlqStats.GetError()) + assert.Equal(t, int64(1), dlqStats.GetMessageCount()) +} + +func TestDeduplication(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dedup", Subjects: []string{"e"}} + dup := func() *pb.Message { + return &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} + } + require.Nil(t, p.PublishOne(ctx, dup())) + require.Nil(t, p.PublishOne(ctx, dup())) + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dedup.consumer", + Address: &pb.Address{Name: "events.dedup"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(1), stats.GetMessageCount(), "duplicate publish_id within the window is collapsed") +} + +func TestHeaderFilterDropsNonMatching(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.filter", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("keep"), Headers: map[string]string{"region": "us"}})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("drop"), Headers: map[string]string{"region": "eu"}})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.filter.consumer", "events.filter", "e") + src.Filters = []*pb.Filter{{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "region", Value: "us"}}, + }} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + assert.Equal(t, "keep", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // The non-matching message is acked and dropped proxy-side, so nothing else + // should arrive. + select { + case extra := <-out: + t.Fatalf("unexpected second delivery: %q", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + +func TestDurableConsumerResumesBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.durable", Subjects: []string{"e"}} + src := queueSource("events.durable.consumer", "events.durable", "e") + + // Establish the durable consumer, then stop consuming (client "outage"). + declCtx, declCancel := context.WithCancel(ctx) + go p.Subscribe(declCtx, src, make(chan *pb.Message, 1)) + time.Sleep(200 * time.Millisecond) + declCancel() + time.Sleep(100 * time.Millisecond) + + // Publish a backlog while no consumer is attached. + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("b%d", i))})) + } + + // Reattach the same durable; the backlog is redelivered. + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + for i := 0; i < n; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + +func TestDeclareOnly(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := queueSource("events.declare.consumer", "events.declare", "e") + src.DeclareOnly = true + + // DeclareOnly establishes topology and returns without blocking. + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(ctx, src, make(chan *pb.Message)) }() + select { + case perr := <-done: + assert.Nil(t, perr) + case <-time.After(10 * time.Second): + t.Fatal("DeclareOnly Subscribe did not return") + } + + // The stream exists afterwards. + stats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "events.declare"}}) + assert.Nil(t, stats.GetError()) +} + +func TestAckUnknownUUID(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + perr := p.Ack(ctx, "does-not-exist") + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "no message with uuid") +} + +func TestStreamConfigFromEnv(t *testing.T) { + t.Setenv("NATSJS_STREAM_REPLICAS", "5") + assert.Equal(t, 5, streamReplicas()) + t.Setenv("NATSJS_STREAM_REPLICAS", "bad") + assert.Equal(t, 1, streamReplicas()) + + t.Setenv("NATSJS_STREAM_MAX_AGE", "24h") + assert.Equal(t, 24*time.Hour, streamMaxAge()) + t.Setenv("NATSJS_STREAM_MAX_AGE", "0") + assert.Equal(t, time.Duration(0), streamMaxAge()) + + t.Setenv("NATSJS_STREAM_MAX_BYTES", "1048576") + assert.Equal(t, int64(1048576), streamMaxBytes()) + t.Setenv("NATSJS_STREAM_MAX_BYTES", "0") + assert.Equal(t, int64(0), streamMaxBytes()) +} + +func TestDeliverPolicyFor(t *testing.T) { + mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } + assert.Equal(t, jetstream.DeliverAllPolicy, deliverPolicyFor(mk("first"))) + assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk("next"))) + assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk(""))) + assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk("garbage"))) + assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(&pb.Source{})) +} + +func TestFirstSubject(t *testing.T) { + assert.Equal(t, "created", firstSubject(&pb.Address{Subjects: []string{"created", "ignored"}})) + assert.Equal(t, "", firstSubject(&pb.Address{})) +} + +func TestWaitForConnectNoClient(t *testing.T) { + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + assert.False(t, p.WaitForConnect(ctx)) +} + +func TestSourceStatsMissingStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + stats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "never.created"}}) + assert.NotNil(t, stats.GetError()) +} + +// TestClientIdentifierFailurePropagates drives the error branch every method +// shares: when the client identifier cannot be resolved, each returns an error +// rather than touching the (nil) connection. +func TestClientIdentifierFailurePropagates(t *testing.T) { + saved := GetClientIdentifier + defer func() { GetClientIdentifier = saved }() + GetClientIdentifier = func(context.Context) (string, error) { + return "", fmt.Errorf("no client identifier") + } + + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx := context.Background() + src := queueSource("c", "events.x", "e") + msg := &pb.Message{Address: &pb.Address{Name: "events.x"}} + + assert.NotNil(t, p.Connect(ctx, &pb.ConnectionConfiguration{Host: "127.0.0.1", Port: 1}, false)) + assert.NotNil(t, p.PublishOne(ctx, msg)) + assert.NotNil(t, p.Publish(ctx, make(chan *pb.Message), make(chan *pb.Error))) + assert.NotNil(t, p.Subscribe(ctx, src, make(chan *pb.Message))) + assert.NotNil(t, p.Ack(ctx, "x")) + assert.NotNil(t, p.Nack(ctx, "x")) + assert.NotNil(t, p.Retry(ctx, src, "x", 1)) + assert.NotNil(t, p.DeadLetter(ctx, src, "x")) + assert.False(t, p.WaitForConnect(ctx)) + assert.NotNil(t, p.SourceStats(ctx, src).GetError()) + p.Disconnect(ctx) // exercises the early-return path; must not panic +} + +// errOf adapts a *pb.Error into a Go error for assert.NoError-style checks. +func errOf(e *pb.Error) error { + if e == nil { + return nil + } + return fmt.Errorf("%s", e.GetMessage()) +} From 1af8ff87f9446fcf88fb081c82812b84763731b7 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Mon, 29 Jun 2026 13:41:28 -0400 Subject: [PATCH 02/77] docs: add NATS JetStream connector design notes Document the natsjs connector: subject/topology mapping, durable vs ephemeral consumer selection, the ack/retry/dead-letter mapping table, the retry-count header synthesis, deduplication, the work-queue vs append-log retention model and its configuration knobs, a RabbitMQ<->NATS feature-parity matrix, supported source options, and known limitations. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 211 +++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 doc/design/natsjs-connector.md diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md new file mode 100644 index 0000000..22c811b --- /dev/null +++ b/doc/design/natsjs-connector.md @@ -0,0 +1,211 @@ +# NATS JetStream Connector + +## Purpose + +This document describes the `natsjs` connector, a backend provider that lets +Arke run against [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) +in place of (or alongside) RabbitMQ / AMQP 0.9.1. It is a worked example of +the contract in +[provider-connector-interface.md](provider-connector-interface.md): a second +`provider.Provider` registered under the name `natsjs` and selected per +connection via `ConnectionConfiguration.provider`. + +The connector covers the publish / subscribe / ack / delayed-retry / +dead-letter / dedup / header-filter paths an AMQP client drives through Arke. +It maps each onto a native JetStream primitive where one exists and onto a +small amount of stateless proxy-side translation where it does not. + +The guiding rule that keeps the translation honest: + +> Arke may do stateless translation and orchestration of broker primitives. It +> must not become the system of record. Persistence, routing, and HA stay in +> the broker; dead-letter orchestration, delayed-retry, retry-count +> bookkeeping, and header filtering may move into the proxy. + +## Subject and topology mapping + +AMQP routes via an exchange plus routing-key bindings. NATS routes via subjects +with token wildcards. The connector maps the two as follows +(`helpers.go:subjectFor`): + +| AMQP concept | NATS mapping | +| --- | --- | +| exchange / address name | subject root (dots kept; it is just a prefix) | +| routing key | appended subject tokens | +| `#` (zero-or-more words) | `>` (NATS allows `>` only as the final token) | +| `*` (exactly one word) | `*` | + +A JetStream stream named `arke_
` captures `
.>`, and each +consumer filters on the mapped source subjects. This is a faithful mapping for +the dotted, topic-style routing keys AMQP topic exchanges use. + +NATS subjects are stricter than AMQP routing keys, so `translateWildcards` +also sanitizes each token: a non-terminal `#` becomes `*` (NATS `>` is +tail-only), illegal characters (space, tab, `*`, `>`) inside a literal token +become `_`, and empty tokens (from `a..b` or a trailing dot) are dropped so the +result is always a valid subject. + +AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced +proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a +separate binding), and within a single filter the matches combine per +`Filter.Type` (`ALL` = and, `ANY` = or). + +## Consumers: durable vs ephemeral + +The connector chooses the consumer kind from the source +(`helpers.go:durableName`): + +- Durable (work-queue) consumers use a stable name and persist across client + reconnects, so a backlog published while the consumer is disconnected is + redelivered on reconnect, matching RabbitMQ durable-queue semantics. Used + for non-transient `QUEUE` sources, `STREAM` sources with a `ConsumerGroup`, + and any `SingleActiveConsumer`. +- Ephemeral consumers auto-expire after an inactivity threshold. Used for + auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use + for per-instance, transient subscriptions. + +The `DeliverPolicy` is taken from the `Offset` option (`first` -> deliver all, +otherwise deliver new) and only applies on first creation; a reconnecting +durable resumes from its stored ack position. + +## Ack, retry, and dead-letter mapping + +The server translates a client's ack / nack / requeue into provider calls; the +connector maps those onto JetStream primitives: + +| Client action | Provider call | NATS JetStream primitive | +| --- | --- | --- | +| ack | `Ack` | `msg.Ack()` | +| nack, `requeue_delay > 0` | `Retry(delay)` | `msg.NakWithDelay(delay)` | +| nack, `requeue_delay == 0`, DLA set | `DeadLetter` | publish to DLQ + `Term()` | +| nack, `requeue_delay == 0`, no DLA | `Nack` | `msg.Nak()` | + +`NakWithDelay` is the native replacement for RabbitMQ's per-message-TTL + +dead-letter retry-queue idiom; JetStream increments the delivery count, which +the connector surfaces back to the client as the `x-retry-count` header (see +below). JetStream has no native dead-letter exchange, so `DeadLetter` +republishes the message to the configured dead-letter subject and then +`Term()`s it to stop redelivery. + +## Retry-count header + +Clients that count retries via an `x-retry-count` header (rather than +RabbitMQ's `x-death`) work unchanged: `handleDelivery` synthesizes +`x-retry-count` from JetStream's `NumDelivered` metadata. This is the key piece +that lets an existing AMQP retry policy run against NATS with no `x-death` +equivalent. + +## Deduplication + +Publish-side dedup maps onto JetStream's `Nats-Msg-Id` plus the stream +`Duplicates` window. When a message carries a publisher name and/or publish id, +the connector sets `Nats-Msg-Id` so re-publishes within the window are +collapsed. + +## Retention: work-queue vs append-log + +This is the most important behavioral difference between the two brokers. A +RabbitMQ queue is a work queue: a message is deleted the instant a consumer +acks it. A JetStream stream is an append log: with the default `LimitsPolicy` +an acked message stays in the log and is only evicted when a configured limit +(`MaxAge`, `MaxBytes`, `MaxMsgs`) is reached. + +To stop a stream growing without bound, `ensureStream` sets: + +- `MaxAge` (default 72h, override with `NATSJS_STREAM_MAX_AGE`) — the time + guard. It is the natural map for AMQP `MessageTTL` / `Expires` (both + durations). It is mutable, so it also reins in streams created before a limit + existed. +- `MaxBytes` (default unlimited, override with `NATSJS_STREAM_MAX_BYTES`) — an + optional hard storage cap. With `discard=old` it only evicts near the cap, so + recent backlog is preserved as long as possible. + +There is a tension to respect: `MaxAge` evicts purely by clock, even with free +disk. If it is shorter than the longest tolerable consumer-outage window it +would silently delete a down consumer's backlog and regress the durable-queue +behavior. The default therefore exceeds any realistic outage, and `MaxBytes` +is intended as the primary disk guard. + +## Configuration + +| Environment variable | Default | Meaning | +| --- | --- | --- | +| `NATSJS_STREAM_REPLICAS` | `1` | Stream replication factor. Set to `3` against a clustered server for the HA equivalent of quorum queues. | +| `NATSJS_STREAM_MAX_AGE` | `72h` | Max age before messages are evicted (Go duration; `0` = keep forever). | +| `NATSJS_STREAM_MAX_BYTES` | `0` (unlimited) | Hard per-stream storage cap in bytes. | + +TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, +`Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP +connector; the broker certificate is verified against the system trust store. + +## Feature-parity matrix + +Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; +**Drop** = not carried forward. + +| RabbitMQ / Arke feature | Disposition | Notes | +| --- | --- | --- | +| Topic routing + wildcard bindings | Native | NATS subjects + `*`/`>`; `#`->`>`, `*`->`*`. | +| Per-subscriber ephemeral queue | Native | Ephemeral consumer with inactivity threshold. | +| Durable work queue (reconnect resumes backlog) | Native | Durable consumer for non-transient / consumer-group / single-active sources. | +| Publish confirms | Native | `js.PublishMsg` returns a `PubAck`. | +| Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | +| Streams: offsets, start position | Native | JetStream is a log; `DeliverPolicy` maps `Offset`. | +| Single active consumer | Native | Consumer config (needs a durable name). | +| Prefetch / QoS | Native | `MaxAckPending`. | +| HA / quorum queues | Native | JetStream R3 (Raft) via `NATSJS_STREAM_REPLICAS`. | +| Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | +| Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | +| Dead-letter (DLX) | Proxy | No native DLX; republish to DLQ subject then `Term()`. | +| Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters`. | +| `MessageTTL` / `Expires` | Partial | Mapped onto stream-level `MaxAge` (see limitations). | +| Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | +| Publish / deliver rates in stats | Drop | Not exposed by JetStream info; would need sampling. | +| RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | + +## Source options + +Per the connector-interface contract, `SupportedSourceOptions()` advertises +every `Source.Options` key the connector reads: + +| Key | Type | Description | +| --- | --- | --- | +| `MessageTTL` | string (ms) | Per-source message TTL (mapped to stream `MaxAge`). | +| `Expires` | string (ms) | Source expiry when unused (mapped to stream `MaxAge`). | +| `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | +| `DeadLetterSubject` | string | Routing key for dead-lettered messages. | +| `Offset` | string | Stream starting offset (`first` / `next`). | +| `ConsumerGroup` | string | Durable consumer group name (Streams only). | + +## Known limitations + +- **Per-source TTL fidelity.** The connector uses one stream per address + root, so `MessageTTL` / `Expires` can only be honored as a stream-wide + `MaxAge`, not a faithful per-source TTL. True per-source TTL (or switching + queue sources to a delete-on-ack policy such as `WorkQueuePolicy` / + `InterestPolicy`) needs a per-source stream topology, and `Retention` is + immutable, so that is a stream-recreate migration rather than an in-place + change. +- **Publish / deliver rates** are not exposed by JetStream stream info, so + `SourceStats` leaves those rate fields at zero. +- **Dead-letter is fire-and-forget.** Messages are republished to the + dead-letter subject; there is no advisory-driven re-consumption. +- **Connection authentication.** The connector supports user/password and TLS + today; NKEYs / JWT auth are a natural follow-up for production deployments. + +## Testing + +Unit tests for the pure mapping logic (subject / wildcard translation, +durable-name selection, header-filter evaluation) live in `helpers_test.go`. +Behavioral tests that exercise publish / subscribe / ack / retry / dead-letter +against a real broker should follow the integration-test guidance in +[provider-connector-interface.md](provider-connector-interface.md) by adding a +NATS service to the integration compose setup. + +## Migration approach + +Run `natsjs` as a second provider alongside `amqp091` — the `provider` field +in `ConnectionConfiguration` already supports per-connection selection. Migrate +one low-risk message class, measure cost and latency, then expand. Because the +proxy absorbs the broker-specific translation, the migration is incremental and +reversible, and there is no need to remove the AMQP connector to begin. From 6f4e4a9f114c8a494ee9cf56f09168893c6e6afe Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Thu, 2 Jul 2026 00:12:48 -0400 Subject: [PATCH 03/77] feat: harden natsjs topology creation and consumer liveness Bound JetStream management API calls (stream and consumer creation) with an explicit deadline (NATSJS_API_TIMEOUT, default 30s) instead of the client library's 5s default, which can expire while a replicated stream is still forming its raft group on cold or network-attached storage. Collapse concurrent ensureStream calls for the same stream across all client connections into a single CreateOrUpdateStream, so a mass reconnect no longer issues one redundant creation call per connection against the JetStream metadata leader. Failures are not cached and success is still memoized per connection. Run consumers with an explicit 5s idle heartbeat and a consume error handler: a stalled delivery path is now logged at warn level and the pull request is re-issued after ~2x the heartbeat, instead of failing silently on the library defaults. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 32 +++- internal/provider/connectors/natsjs/natsjs.go | 152 +++++++++++++++--- .../provider/connectors/natsjs/natsjs_test.go | 83 ++++++++++ 3 files changed, 242 insertions(+), 25 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 22c811b..d05f43a 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -126,6 +126,30 @@ would silently delete a down consumer's backlog and regress the durable-queue behavior. The default therefore exceeds any realistic outage, and `MaxBytes` is intended as the primary disk guard. +## Operational resilience + +Three mechanisms harden the connector against a cold or busy broker. All are +sized for a clustered (replicated) server, where creating topology also means +forming a raft group per stream and per durable consumer: + +- **Bounded topology calls.** Stream and consumer creation carry an explicit + deadline (`NATSJS_API_TIMEOUT`, default 30s) instead of the JetStream + client's built-in 5s default. First-touch creation of a replicated stream + has to finish raft formation and storage allocation before the API call + returns, which can exceed 5s on cold or network-attached storage. +- **Collapsed concurrent creation.** `ensureStream` calls for the same stream + are collapsed provider-wide: when many clients (re)connect at once — a + proxy restart, a mass reconnect after a broker outage — exactly one + `CreateOrUpdateStream` per stream is in flight at a time, and concurrent + callers share its result. Failures are not cached, and success is still + memoized per connection, so a fresh connection re-asserts its topology. +- **Consumer liveness.** Consumers run with a 5s idle heartbeat and a consume + error handler. If the server stops serving a consumer's pulls (a broker + restart, or a just-created consumer whose raft group is not yet serving), + the missed heartbeat is logged at warn level and the client re-issues its + pull request after roughly twice the heartbeat. With library defaults the + same stall would go unlogged and take ~30s per detection cycle. + ## Configuration | Environment variable | Default | Meaning | @@ -133,6 +157,7 @@ is intended as the primary disk guard. | `NATSJS_STREAM_REPLICAS` | `1` | Stream replication factor. Set to `3` against a clustered server for the HA equivalent of quorum queues. | | `NATSJS_STREAM_MAX_AGE` | `72h` | Max age before messages are evicted (Go duration; `0` = keep forever). | | `NATSJS_STREAM_MAX_BYTES` | `0` (unlimited) | Hard per-stream storage cap in bytes. | +| `NATSJS_API_TIMEOUT` | `30s` | Deadline for JetStream management API calls (stream / consumer creation). Go duration. | TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, `Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP @@ -197,10 +222,9 @@ every `Source.Options` key the connector reads: Unit tests for the pure mapping logic (subject / wildcard translation, durable-name selection, header-filter evaluation) live in `helpers_test.go`. -Behavioral tests that exercise publish / subscribe / ack / retry / dead-letter -against a real broker should follow the integration-test guidance in -[provider-connector-interface.md](provider-connector-interface.md) by adding a -NATS service to the integration compose setup. +Behavioral tests in `natsjs_test.go` drive the publish / subscribe / ack / +retry / dead-letter / dedup paths against an in-process JetStream-enabled +`nats-server`, so they run without an external broker. ## Migration approach diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index c7f2a57..f5a2b45 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -43,6 +43,17 @@ const ( defaultInactiveThreshold = 5 * time.Minute defaultDedupWindow = 2 * time.Minute connectPollInterval = 50 * time.Millisecond + // defaultJSAPITimeout bounds JetStream management API calls (stream and + // consumer creation). Without an explicit deadline the JetStream client + // applies a 5s default, which can expire while a replicated stream is + // still forming its raft group on cold or network-attached storage. + // Override via NATSJS_API_TIMEOUT. + defaultJSAPITimeout = 30 * time.Second + // defaultConsumeHeartbeat is the pull-consumer idle heartbeat. If the + // delivery path stalls, the missed heartbeat is surfaced through the + // consume error handler and the client re-issues its pull request after + // roughly twice this interval, instead of ~30s with the library defaults. + defaultConsumeHeartbeat = 5 * time.Second // defaultStreamMaxAge bounds how long a stream retains messages so the // JetStream log does not grow without bound (RabbitMQ deletes on ack; // JetStream LimitsPolicy does not). 72h is generous enough not to truncate a @@ -84,6 +95,58 @@ type natsBrokerDetails struct { type natsjsProvider struct { connections *util.ConcurrentMap + // streams collapses concurrent stream-creation calls across connections. + streams *streamRegistry +} + +// streamRegistry collapses concurrent ensureStream calls for the same stream +// into a single JetStream API call. When many clients (re)connect at once — +// e.g. after a broker or proxy restart — every connection would otherwise +// issue its own CreateOrUpdateStream for the same shared streams, piling +// redundant load on the JetStream metadata leader exactly when it is busiest. +// Entries are keyed by broker endpoint + stream name so distinct brokers never +// share results. Only in-flight calls are tracked here; success is memoized +// per connection (knownStreams), preserving the property that a fresh +// connection re-asserts its topology. +type streamRegistry struct { + mu sync.Mutex + inflight map[string]*inflightEnsure +} + +type inflightEnsure struct { + done chan struct{} + err error +} + +func newStreamRegistry() *streamRegistry { + return &streamRegistry{inflight: make(map[string]*inflightEnsure)} +} + +// ensure runs create once per key at a time; callers that arrive while a call +// for the same key is in flight wait for it and share its result. Failures +// are not cached — the next caller retries. +func (r *streamRegistry) ensure(ctx context.Context, key string, create func() error) error { + r.mu.Lock() + if e, ok := r.inflight[key]; ok { + r.mu.Unlock() + select { + case <-e.done: + return e.err + case <-ctx.Done(): + return ctx.Err() + } + } + e := &inflightEnsure{done: make(chan struct{})} + r.inflight[key] = e + r.mu.Unlock() + + e.err = create() + + r.mu.Lock() + delete(r.inflight, key) + r.mu.Unlock() + close(e.done) + return e.err } func init() { @@ -96,7 +159,10 @@ func init() { // NewNATSJetStreamProvider returns a new natsjs provider singleton. func NewNATSJetStreamProvider() provider.Provider { - return &natsjsProvider{connections: util.NewConcurrentMap()} + return &natsjsProvider{ + connections: util.NewConcurrentMap(), + streams: newStreamRegistry(), + } } // streamReplicas controls the JetStream replication factor (1 = single node, @@ -145,6 +211,22 @@ func streamMaxBytes() int64 { return 0 } +// jsAPITimeout bounds JetStream management API calls (CreateOrUpdateStream / +// CreateOrUpdateConsumer). First-touch creation of a replicated stream has to +// finish raft-group formation and storage allocation before the call returns, +// so it can legitimately take longer than the JetStream client's built-in 5s +// default, particularly on network-attached storage. Configured via +// NATSJS_API_TIMEOUT as a Go duration ("30s", "1m"); defaults to +// defaultJSAPITimeout. +func jsAPITimeout() time.Duration { + if v := os.Getenv("NATSJS_API_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultJSAPITimeout +} + func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { clientID, err := GetClientIdentifier(ctx) if err != nil { @@ -279,28 +361,39 @@ func (p *natsjsProvider) Disconnect(ctx context.Context) { } // ensureStream lazily creates (or updates) a JetStream stream that captures all -// subjects under the given address root. +// subjects under the given address root. Concurrent calls for the same stream +// — across all client connections to the same broker — are collapsed into one +// API call (see streamRegistry). That call gets an explicit deadline +// (jsAPITimeout) and is detached from the triggering caller's cancellation: +// the stream is shared topology and other callers may be waiting on the same +// result. func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails, addressName string) (string, error) { streamName := streamNameFor(addressName) if _, ok := bd.knownStreams.Get(streamName); ok { return streamName, nil } - _, err := bd.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: streamSubjectsFor(addressName), - Storage: jetstream.FileStorage, - Retention: jetstream.LimitsPolicy, - Duplicates: defaultDedupWindow, - // Replicas via NATSJS_STREAM_REPLICAS (default 1). Set to 3 against a - // clustered nats-server for the HA-equivalent of RabbitMQ quorum queues. - Replicas: streamReplicas(), - // MaxAge/MaxBytes bound the log so it does not grow without limit the way - // the default LimitsPolicy otherwise would (RabbitMQ deletes on ack; - // JetStream retains acked messages). MaxAge is the time guard, MaxBytes an - // optional hard storage cap. Both are mutable, so this also reins in - // streams created before the limits existed. See natsjs-connector.md. - MaxAge: streamMaxAge(), - MaxBytes: streamMaxBytes(), + key := fmt.Sprintf("%s:%d/%s", bd.connectionConfig.GetHost(), bd.connectionConfig.GetPort(), streamName) + err := p.streams.ensure(ctx, key, func() error { + cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jsAPITimeout()) + defer cancel() + _, err := bd.js.CreateOrUpdateStream(cctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: streamSubjectsFor(addressName), + Storage: jetstream.FileStorage, + Retention: jetstream.LimitsPolicy, + Duplicates: defaultDedupWindow, + // Replicas via NATSJS_STREAM_REPLICAS (default 1). Set to 3 against a + // clustered nats-server for the HA-equivalent of RabbitMQ quorum queues. + Replicas: streamReplicas(), + // MaxAge/MaxBytes bound the log so it does not grow without limit the way + // the default LimitsPolicy otherwise would (RabbitMQ deletes on ack; + // JetStream retains acked messages). MaxAge is the time guard, MaxBytes an + // optional hard storage cap. Both are mutable, so this also reins in + // streams created before the limits existed. See natsjs-connector.md. + MaxAge: streamMaxAge(), + MaxBytes: streamMaxBytes(), + }) + return err }) if err != nil { return "", err @@ -381,7 +474,13 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if serr != nil { return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", serr.Error()), IsFatal: true} } - stream, serr := bd.js.Stream(ctx, streamName) + // The management API calls below get the same explicit deadline as stream + // creation (see jsAPITimeout): creating a replicated consumer forms its own + // raft group, which can outlast the client library's 5s default while the + // server is cold or busy. + tctx, tcancel := context.WithTimeout(ctx, jsAPITimeout()) + defer tcancel() + stream, serr := bd.js.Stream(tctx, streamName) if serr != nil { return &pb.Error{Message: fmt.Sprintf("open stream: %s", serr.Error()), IsFatal: true} } @@ -409,7 +508,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c consCfg.InactiveThreshold = defaultInactiveThreshold } - cons, cerr := stream.CreateOrUpdateConsumer(ctx, consCfg) + cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) if cerr != nil { // Include the mapped filter subjects: a NATS "invalid subject" (10052) // otherwise gives no hint which source/subject produced it. @@ -426,7 +525,18 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c cc, cerr := cons.Consume(func(m jetstream.Msg) { p.handleDelivery(ctx, bd, source, m, out) - }) + }, + // A short idle heartbeat plus an error handler keep a stalled delivery + // path from failing silently: if the server stops serving this + // consumer's pulls (broker restart, consumer raft leader not yet + // serving after creation), the missed heartbeat is logged at warn and + // the library re-issues its pull request after ~2x the heartbeat, + // instead of ~30s — and invisibly — with the defaults. + jetstream.PullHeartbeat(defaultConsumeHeartbeat), + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + util.Logger.Warn("natsjs: consume error on source {0}: {1}", source.GetName(), err.Error()) + }), + ) if cerr != nil { return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 03a4f42..49c6d93 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net" + "sync/atomic" "testing" "time" @@ -353,6 +354,88 @@ func TestStreamConfigFromEnv(t *testing.T) { assert.Equal(t, int64(0), streamMaxBytes()) } +func TestJSAPITimeoutFromEnv(t *testing.T) { + t.Setenv("NATSJS_API_TIMEOUT", "45s") + assert.Equal(t, 45*time.Second, jsAPITimeout()) + t.Setenv("NATSJS_API_TIMEOUT", "bad") + assert.Equal(t, defaultJSAPITimeout, jsAPITimeout()) + t.Setenv("NATSJS_API_TIMEOUT", "-1s") + assert.Equal(t, defaultJSAPITimeout, jsAPITimeout()) +} + +func TestStreamRegistryCollapsesConcurrentCalls(t *testing.T) { + r := newStreamRegistry() + release := make(chan struct{}) + leaderStarted := make(chan struct{}) + const key = "broker:4222/arke_events" + + // One call gets in flight and blocks... + leaderErr := make(chan error, 1) + go func() { + leaderErr <- r.ensure(context.Background(), key, func() error { + close(leaderStarted) + <-release + return nil + }) + }() + <-leaderStarted + + // ...then every caller that arrives while it is in flight must piggyback + // on it instead of running its own create. + var extraCalls int32 + const n = 8 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func() { + errs <- r.ensure(context.Background(), key, func() error { + atomic.AddInt32(&extraCalls, 1) + return nil + }) + }() + } + time.Sleep(250 * time.Millisecond) // let the followers reach ensure + close(release) + + assert.NoError(t, <-leaderErr) + for i := 0; i < n; i++ { + assert.NoError(t, <-errs) + } + assert.Equal(t, int32(0), atomic.LoadInt32(&extraCalls), "followers share the in-flight result") +} + +func TestStreamRegistryDoesNotCacheFailures(t *testing.T) { + r := newStreamRegistry() + boom := fmt.Errorf("create failed") + assert.ErrorIs(t, r.ensure(context.Background(), "k", func() error { return boom }), boom) + + calls := 0 + assert.NoError(t, r.ensure(context.Background(), "k", func() error { calls++; return nil })) + assert.Equal(t, 1, calls, "a failed create is retried by the next caller") +} + +func TestStreamRegistryFollowerHonorsContext(t *testing.T) { + r := newStreamRegistry() + release := make(chan struct{}) + started := make(chan struct{}) + go func() { + _ = r.ensure(context.Background(), "k", func() error { + close(started) + <-release + return nil + }) + }() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := r.ensure(ctx, "k", func() error { + t.Error("follower must not run create") + return nil + }) + assert.ErrorIs(t, err, context.Canceled) + close(release) +} + func TestDeliverPolicyFor(t *testing.T) { mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } assert.Equal(t, jetstream.DeliverAllPolicy, deliverPolicyFor(mk("first"))) From 0da7cd003b465039f16b24e7f3f1bdd1c2d81fc3 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Thu, 2 Jul 2026 17:22:39 -0400 Subject: [PATCH 04/77] fix: keep streams for prefix-related addresses disjoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each address gets its own stream, and JetStream requires stream subject spaces to be disjoint — but address names are dotted, so two addresses can sit in a prefix relationship (events.orders and events.orders.filtered). Their capture wildcards then overlap: whichever stream forms first wins and every publish/subscribe on the other address fails forever with 'subjects overlap with an existing stream' (10065), retried by clients in a hot loop. Fix by reserving a delimiter token '~' between the address root and the routing-key tokens, and stripping '~' from address tokens, so any two distinct roots map to disjoint subject spaces by construction. This also stops cross-address leakage (routing key 'filtered.x' on events.orders was previously indistinguishable from 'x' on events.orders.filtered), gives empty routing keys a concrete publishable subject (.~ instead of the wildcard .>), replaces the empty address's capture-everything '>' subject, and sanitizes wildcard characters out of published routing keys (literal in AMQP publishes, forbidden by NATS). Binding patterns with a trailing '#' now also match the zero-word case, as AMQP does. Existing data stored under the old subject scheme no longer matches consumer filters after the stream's subjects move forward; see the migration note in doc/design/natsjs-connector.md. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 52 ++++-- .../provider/connectors/natsjs/helpers.go | 168 +++++++++++++----- .../connectors/natsjs/helpers_test.go | 131 +++++++++++--- internal/provider/connectors/natsjs/natsjs.go | 4 +- .../provider/connectors/natsjs/natsjs_test.go | 80 +++++++++ 5 files changed, 351 insertions(+), 84 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index d05f43a..6d3d9dd 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -26,24 +26,54 @@ The guiding rule that keeps the translation honest: AMQP routes via an exchange plus routing-key bindings. NATS routes via subjects with token wildcards. The connector maps the two as follows -(`helpers.go:subjectFor`): +(`helpers.go:publishSubjectFor` / `filterSubjectsFor`): | AMQP concept | NATS mapping | | --- | --- | -| exchange / address name | subject root (dots kept; it is just a prefix) | +| exchange / address name | subject root (dots kept; it is a prefix) | +| — | `~` delimiter token, always inserted after the root | | routing key | appended subject tokens | | `#` (zero-or-more words) | `>` (NATS allows `>` only as the final token) | | `*` (exactly one word) | `*` | -A JetStream stream named `arke_
` captures `
.>`, and each -consumer filters on the mapped source subjects. This is a faithful mapping for -the dotted, topic-style routing keys AMQP topic exchanges use. - -NATS subjects are stricter than AMQP routing keys, so `translateWildcards` -also sanitizes each token: a non-terminal `#` becomes `*` (NATS `>` is -tail-only), illegal characters (space, tab, `*`, `>`) inside a literal token -become `_`, and empty tokens (from `a..b` or a trailing dot) are dropped so the -result is always a valid subject. +A message published to address `events.orders` with routing key +`region.us.created` travels on subject `events.orders.~.region.us.created`; +an empty routing key maps to the bare prefix `events.orders.~`. A JetStream +stream named `arke_
` captures `
.~` and `
.~.>`, and +each consumer filters on the mapped source subjects. + +The `~` delimiter is what keeps distinct addresses' streams disjoint, which +JetStream requires: two subjects may belong to at most one stream. Address +names are themselves dotted, so two addresses can sit in a prefix +relationship (`events.orders` and `events.orders.filtered`); without the +delimiter their capture wildcards overlap, whichever stream is created first +wins, and the other address fails every publish and subscribe with `subjects +overlap with an existing stream` (err 10065). With it, the token after the +shared prefix differs (`~` vs `filtered`) and sanitization guarantees no +address token ever equals `~`, so any two distinct roots are disjoint. The +delimiter also prevents cross-address message leakage — without it, +publishing routing key `filtered.x` to `events.orders` is indistinguishable +from publishing `x` to `events.orders.filtered`. + +NATS subjects are stricter than AMQP routing keys, so each token is also +sanitized. In binding patterns (`translateWildcards`): a non-terminal `#` +becomes `*` (NATS `>` is tail-only), illegal characters (space, tab, `*`, +`>`) inside a literal token become `_`, and empty tokens (from `a..b` or a +trailing dot) are dropped. A pattern whose trailing `#` became `>` also gets +the zero-word variant as a second filter subject (AMQP `#` matches zero or +more words, NATS `>` one or more, so binding `a.#` must match routing key +`a`). On the publish side routing keys are literal — AMQP gives `*`/`#` +meaning only in bindings, and NATS forbids wildcard tokens in published +subjects — so wildcard characters are sanitized to `_` like any other illegal +character. In address names, empty tokens and tokens equal to `~` are +replaced with `_` rather than dropped, so distinct names keep distinct roots. + +Changing the subject scheme is a breaking change for data already stored +under an older scheme: `CreateOrUpdateStream` moves the stream's captured +subjects forward, after which messages persisted under old-style subjects no +longer match any consumer filter and age out via the retention limits. Drain +consumers (or recreate streams) when upgrading a deployment that has retained +backlog. AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 2e785c8..c03f342 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -15,49 +15,101 @@ import ( // AMQP (RabbitMQ) routes via exchange + routing-key bindings. NATS routes via // subjects with token wildcards. We map: // -// exchange/address name -> subject root (dots kept; it is just a multi-token prefix) +// exchange/address name -> subject root (dots kept; it is a multi-token prefix) +// "~" -> reserved delimiter token, always inserted after the root // routing key -> appended subject tokens // AMQP '#' (zero+ words) -> NATS '>' (NATS only allows '>' as the final token) // AMQP '*' (one word) -> NATS '*' // -// A JetStream stream captures ".>" and consumers filter on the mapped -// source subjects. This is a faithful mapping for the dotted, topic-style keys -// AMQP topic exchanges use (e.g. orders.region.us.created.*). It does NOT -// reproduce AMQP headers-exchange routing — that is handled proxy-side in -// evaluateFilters (see Subscribe). +// A message published to address "events.orders" with routing key +// "region.us.created" travels on subject "events.orders.~.region.us.created". +// A JetStream stream captures ".~" and ".~.>" and consumers filter +// on the mapped source subjects. +// +// The "~" delimiter exists because each address gets its own stream, and +// JetStream requires every stream's subject space to be disjoint. Address +// names themselves are dotted, so two addresses can sit in a prefix +// relationship — "events.orders" and "events.orders.filtered" — and without a +// delimiter their capture wildcards ("events.orders.>" vs +// "events.orders.filtered.>") overlap: whichever stream is created first wins +// and the other fails forever with "subjects overlap with an existing stream" +// (err 10065). The delimiter makes any two distinct roots disjoint: the token +// after "events.orders" is "~" for its own traffic but "filtered" for the +// other address, and "~" is stripped from address tokens by sanitization, so +// no address token can ever equal the delimiter. It also keeps the two +// addresses' messages apart (without it, publishing "filtered.x" to +// "events.orders" would be indistinguishable from publishing "x" to +// "events.orders.filtered"), and gives the empty routing key a concrete +// subject (".~"). +// +// This is a faithful mapping for the dotted, topic-style keys AMQP topic +// exchanges use (e.g. orders.region.us.created.*). It does NOT reproduce AMQP +// headers-exchange routing — that is handled proxy-side in evaluateFilters +// (see Subscribe). + +// addressDelim is the reserved token inserted between the address root and the +// routing-key tokens. Sanitization guarantees no address token equals it. +const addressDelim = "~" + +// addressRoot sanitizes an address name into a valid dotted subject prefix. +// Empty tokens and tokens equal to the reserved delimiter are replaced rather +// than dropped so distinct address names keep distinct roots where possible. +func addressRoot(addressName string) string { + if addressName == "" { + return "" + } + tokens := strings.Split(addressName, ".") + for i, t := range tokens { + t = tokenSanitizer.Replace(t) + if t == "" || t == addressDelim { + t = "_" + } + tokens[i] = t + } + return strings.Join(tokens, ".") +} + +// subjectPrefix returns the subject an empty routing key maps to, which is also +// the prefix of every subject under the address. +func subjectPrefix(addressName string) string { + if root := addressRoot(addressName); root != "" { + return root + "." + addressDelim + } + return addressDelim +} // streamNameFor derives a JetStream-legal stream name from an address name. -// Stream names may not contain '.', ' ', '*' or '>'. +// Stream names may not contain '.', ' ', '*' or '>'. Derived from the +// sanitized root so every address that shares a root shares a stream. func streamNameFor(addressName string) string { - r := strings.NewReplacer(".", "_", " ", "_", "*", "_", ">", "_") - return "arke_" + r.Replace(addressName) + return "arke_" + strings.ReplaceAll(addressRoot(addressName), ".", "_") } -// subjectFor joins an address root with a routing key / pattern, translating -// AMQP wildcards to NATS wildcards. -func subjectFor(addressName, routingKey string) string { - root := addressName - if routingKey == "" { - if root == "" { - return ">" - } - return root + ".>" - } - rk := translateWildcards(routingKey) - if rk == "" { - // The routing key was made up entirely of empty tokens (e.g. ".", - // trailing/leading dots): treat it as capture-all under the root. - if root == "" { - return ">" +// publishSubjectFor maps an address + routing key onto the concrete subject a +// message is published to. Routing keys are literal on the publish side — +// AMQP only gives '*' / '#' wildcard meaning in bindings — and NATS forbids +// wildcard tokens in published subjects, so they are sanitized like any other +// illegal character. +func publishSubjectFor(addressName, routingKey string) string { + prefix := subjectPrefix(addressName) + tokens := strings.Split(routingKey, ".") + out := make([]string, 0, len(tokens)) + for _, t := range tokens { + if t == "" { + continue } - return root + ".>" + out = append(out, publishSanitizer.Replace(t)) } - if root == "" { - return rk + if len(out) == 0 { + return prefix } - return root + "." + rk + return prefix + "." + strings.Join(out, ".") } +// publishSanitizer is tokenSanitizer plus '#': on the publish side wildcard +// characters carry no meaning and must not appear in the subject. +var publishSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_", "#", "_") + // tokenSanitizer replaces characters that are illegal inside a NATS subject // token (space/tab, and the wildcard chars '*'/'>') so a literal AMQP token that // happens to embed one of them still yields a valid subject. @@ -77,7 +129,7 @@ var tokenSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_" // other -> illegal chars (space/tab/'*'/'>') replaced with '_' // // The result is always a valid NATS subject fragment (possibly empty if every -// token was empty; subjectFor handles that). +// token was empty; filterSubjectsFor handles that). func translateWildcards(key string) string { tokens := strings.Split(key, ".") out := make([]string, 0, len(tokens)) @@ -100,25 +152,51 @@ func translateWildcards(key string) string { return strings.Join(out, ".") } -// streamSubjectsFor returns the wildcard subject set a stream should capture for -// the given address. +// streamSubjectsFor returns the subject set a stream captures for the given +// address: the bare prefix (empty routing key) and everything under it. The +// two are disjoint from each other ('>' needs at least one more token) and, +// thanks to the delimiter, from every other address's set. func streamSubjectsFor(addressName string) []string { - if addressName == "" { - return []string{">"} - } - return []string{addressName + ".>"} + prefix := subjectPrefix(addressName) + return []string{prefix, prefix + ".>"} } -// filterSubjectsFor maps a source's subjects to NATS consumer filter subjects. +// filterSubjectsFor maps a source's routing-key patterns to NATS consumer +// filter subjects. An empty pattern set means everything under the address. +// A pattern whose AMQP '#' tail became '>' also gets the zero-word variant: +// AMQP '#' matches zero or more words while NATS '>' matches one or more, so +// binding "a.#" must match routing key "a" too. func filterSubjectsFor(source *pb.Source) []string { - addr := source.GetAddress().GetName() - subs := source.GetAddress().GetSubjects() - if len(subs) == 0 { - return []string{subjectFor(addr, "")} - } - out := make([]string, 0, len(subs)) - for _, s := range subs { - out = append(out, subjectFor(addr, s)) + prefix := subjectPrefix(source.GetAddress().GetName()) + pats := source.GetAddress().GetSubjects() + if len(pats) == 0 { + pats = []string{""} + } + out := make([]string, 0, len(pats)+1) + seen := make(map[string]bool, len(pats)+1) + add := func(s string) { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + for _, p := range pats { + pat := translateWildcards(p) + switch { + case pat == "": + add(prefix) + add(prefix + ".>") + case strings.HasSuffix(pat, ">"): + base := strings.TrimSuffix(strings.TrimSuffix(pat, ">"), ".") + if base == "" { + add(prefix) + } else { + add(prefix + "." + base) + } + add(prefix + "." + pat) + default: + add(prefix + "." + pat) + } } return out } diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index fdc98a2..a00c84b 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -4,6 +4,7 @@ package natsjs import ( + "strings" "testing" pb "github.com/sassoftware/arke/api" @@ -15,41 +16,119 @@ func TestStreamNameFor(t *testing.T) { assert.Equal(t, "arke_events_audit", streamNameFor("events.audit")) // illegal subject chars are sanitized assert.Equal(t, "arke_a_b_c", streamNameFor("a.b*c")) + assert.Equal(t, "arke_", streamNameFor("")) } -func TestSubjectFor(t *testing.T) { - // dotted topic key under an exchange root - assert.Equal(t, - "events.orders.region.us.order.created.success", - subjectFor("events.orders", "region.us.order.created.success")) - // AMQP '#' -> NATS '>' - assert.Equal(t, "events.orders.region.>", subjectFor("events.orders", "region.#")) - // AMQP '*' stays '*' - assert.Equal(t, "events.orders.region.*.success", subjectFor("events.orders", "region.*.success")) - // empty routing key -> capture-all under root - assert.Equal(t, "events.orders.>", subjectFor("events.orders", "")) +func TestAddressRoot(t *testing.T) { + assert.Equal(t, "events.orders", addressRoot("events.orders")) + assert.Equal(t, "", addressRoot("")) + // empty tokens are kept as placeholders, not dropped, so "a..b" and "a.b" + // stay distinct roots + assert.Equal(t, "a._.b", addressRoot("a..b")) + // the reserved delimiter token is sanitized out of address names + assert.Equal(t, "a._.b", addressRoot("a.~.b")) + // illegal subject chars inside tokens are replaced + assert.Equal(t, "a.b_c", addressRoot("a.b*c")) } -// TestSubjectForSanitizes covers routing keys that are legal in AMQP but would -// otherwise yield an invalid NATS subject. -func TestSubjectForSanitizes(t *testing.T) { - // non-terminal '#' has no exact NATS equivalent ('>' is tail-only) -> '*' - assert.Equal(t, "events.orders.a.*.b", subjectFor("events.orders", "a.#.b")) - // terminal '#' still maps to '>' - assert.Equal(t, "events.orders.a.b.>", subjectFor("events.orders", "a.b.#")) +func TestPublishSubjectFor(t *testing.T) { + // dotted topic key under an exchange root, behind the delimiter + assert.Equal(t, + "events.orders.~.region.us.order.created.success", + publishSubjectFor("events.orders", "region.us.order.created.success")) + // empty routing key -> the bare prefix (a concrete, publishable subject) + assert.Equal(t, "events.orders.~", publishSubjectFor("events.orders", "")) + // empty address -> subjects live directly under the delimiter + assert.Equal(t, "~.a.b", publishSubjectFor("", "a.b")) + assert.Equal(t, "~", publishSubjectFor("", "")) + // wildcard chars are literal in a published routing key and NATS forbids + // them in publish subjects -> sanitized + assert.Equal(t, "events.orders.~.a._.b", publishSubjectFor("events.orders", "a.#.b")) + assert.Equal(t, "events.orders.~.a._", publishSubjectFor("events.orders", "a.*")) // empty tokens (double dot / trailing dot) are collapsed, not left invalid - assert.Equal(t, "events.orders.a.b", subjectFor("events.orders", "a..b")) - assert.Equal(t, "events.orders.a.b", subjectFor("events.orders", "a.b.")) - // a routing key of only empty tokens -> capture-all under root - assert.Equal(t, "events.orders.>", subjectFor("events.orders", ".")) + assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a..b")) + assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a.b.")) + assert.Equal(t, "events.orders.~", publishSubjectFor("events.orders", ".")) // illegal chars embedded in a literal token are replaced with '_' - assert.Equal(t, "events.orders.a_b.c_d", subjectFor("events.orders", "a*b.c>d")) - assert.Equal(t, "events.orders.a_b", subjectFor("events.orders", "a b")) + assert.Equal(t, "events.orders.~.a_b.c_d", publishSubjectFor("events.orders", "a*b.c>d")) + assert.Equal(t, "events.orders.~.a_b", publishSubjectFor("events.orders", "a b")) } func TestStreamSubjectsFor(t *testing.T) { - assert.Equal(t, []string{"events.orders.>"}, streamSubjectsFor("events.orders")) - assert.Equal(t, []string{">"}, streamSubjectsFor("")) + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + streamSubjectsFor("events.orders")) + assert.Equal(t, []string{"~", "~.>"}, streamSubjectsFor("")) +} + +// TestStreamSubjectsDisjointForPrefixAddresses is the regression test for the +// address-prefix collision: without the delimiter, "events.orders" and +// "events.orders.filtered" produce overlapping stream subjects and the +// second stream can never be created (JetStream err 10065). +func TestStreamSubjectsDisjointForPrefixAddresses(t *testing.T) { + parent := streamSubjectsFor("events.orders") + child := streamSubjectsFor("events.orders.filtered") + for _, p := range parent { + for _, c := range child { + assert.False(t, subjectsOverlap(p, c), "%q overlaps %q", p, c) + } + } + // an address token spelled "~" cannot fake its way into the parent's space + evil := streamSubjectsFor("events.orders.~") + for _, p := range parent { + for _, c := range evil { + assert.False(t, subjectsOverlap(p, c), "%q overlaps %q", p, c) + } + } +} + +// subjectsOverlap reports whether two subject patterns can match a common +// subject (token-wise; '>' matches one or more trailing tokens, '*' exactly +// one). +func subjectsOverlap(a, b string) bool { + at, bt := strings.Split(a, "."), strings.Split(b, ".") + for i := 0; ; i++ { + switch { + case i == len(at) && i == len(bt): + return true + case i == len(at) || i == len(bt): + return false + case at[i] == ">" || bt[i] == ">": + return true + case at[i] == "*" || bt[i] == "*": + continue + case at[i] != bt[i]: + return false + } + } +} + +func TestFilterSubjectsFor(t *testing.T) { + src := func(subjects ...string) *pb.Source { + return &pb.Source{Address: &pb.Address{Name: "events.orders", Subjects: subjects}} + } + + // no patterns -> everything under the address, including the empty key + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src())) + // plain patterns map one to one + assert.Equal(t, + []string{"events.orders.~.created", "events.orders.~.region.*.updated"}, + filterSubjectsFor(src("created", "region.*.updated"))) + // AMQP '#' matches zero or more words -> both the '>' form and the + // zero-word base are included + assert.Equal(t, + []string{"events.orders.~.region", "events.orders.~.region.>"}, + filterSubjectsFor(src("region.#"))) + // a bare '#' pattern behaves like the empty pattern + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#"))) + // duplicates collapse + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#", ""))) } func TestDurableName(t *testing.T) { diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index f5a2b45..c27edf5 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -445,7 +445,7 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} } nmsg := &nats.Msg{ - Subject: subjectFor(addr.GetName(), firstSubject(addr)), + Subject: publishSubjectFor(addr.GetName(), firstSubject(addr)), Data: msg.GetBody(), Header: pbToNatsHeader(msg.GetHeaders()), } @@ -653,7 +653,7 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid if dla := opts["DeadLetterAddress"]; dla != "" { if _, err := p.ensureStream(ctx, bd, dla); err == nil { dlqMsg := &nats.Msg{ - Subject: subjectFor(dla, opts["DeadLetterSubject"]), + Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), Data: m.Data(), Header: m.Headers(), } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 49c6d93..8b8b63d 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -499,3 +499,83 @@ func errOf(e *pb.Error) error { } return fmt.Errorf("%s", e.GetMessage()) } + +// TestPrefixAddressesCoexist is the end-to-end regression test for addresses +// in a dotted-prefix relationship (e.g. "events.jobs" and +// "events.jobs.filter"). Before the "~" delimiter both addresses mapped to +// overlapping stream subjects, so whichever stream was created first won and +// every ensure of the other failed with "subjects overlap with an existing +// stream" (err 10065). Both creation orders are exercised, and traffic on one +// address must not leak into the other even when a routing key spells out the +// other address's name. +func TestPrefixAddressesCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // parent first, then child + parent := &pb.Address{Name: "events.jobs", Subjects: []string{"created"}} + child := &pb.Address{Name: "events.jobs.filter", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: parent, Body: []byte("to-parent")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: child, Body: []byte("to-child")}), + "creating the child stream after the parent must not collide") + + // child first, then parent + child2 := &pb.Address{Name: "events.audit.trail", Subjects: []string{"e"}} + parent2 := &pb.Address{Name: "events.audit", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: child2, Body: []byte("x")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: parent2, Body: []byte("x")}), + "creating the parent stream after the child must not collide") + + // a routing key on the parent that spells the child's name stays on the + // parent's stream + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.jobs", Subjects: []string{"filter.created"}}, + Body: []byte("still-to-parent"), + })) + + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.jobs.filter.consumer", "events.jobs.filter", "created"), out) + + m := recv(t, out) + assert.Equal(t, "to-child", string(m.GetBody()), + "the child consumer sees only the child's message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery: %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + + // the parent consumer sees both parent messages + pout := make(chan *pb.Message, 4) + go p.Subscribe(subCtx, queueSource("events.jobs.consumer", "events.jobs", "#"), pout) + got := map[string]bool{} + for i := 0; i < 2; i++ { + m := recv(t, pout) + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.True(t, got["to-parent"] && got["still-to-parent"], "got %v", got) +} + +// TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no +// routing key: they map to the bare ".~" subject, which both the stream +// and a pattern-less consumer capture. +func TestEmptyRoutingKeyDelivers(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.fanout"} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("no-key")})) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.fanout.consumer", "events.fanout"), out) + + m := recv(t, out) + assert.Equal(t, "no-key", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} From 5c818fb125702c6e40d32aca7175a89e3df60ae2 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Thu, 2 Jul 2026 17:36:45 -0400 Subject: [PATCH 05/77] fix: report durable consumer backlog, not stream depth, in SourceStats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stream retains acked messages under its retention limits (LimitsPolicy), so its message count keeps growing after consumers are caught up. Anything reading SourceStats message_count as queue length — consumer autoscaling in particular — would see a permanently inflated backlog. Report the durable consumer's undelivered + unacked count instead (the amqp091 connector's ready+unacked equivalent), populate current_offset from the consumer's ack floor, and fall back to the stream view for sources without a durable consumer. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 8 +++- internal/provider/connectors/natsjs/natsjs.go | 15 +++++++ .../provider/connectors/natsjs/natsjs_test.go | 39 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 6d3d9dd..1841a29 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -242,7 +242,13 @@ every `Source.Options` key the connector reads: immutable, so that is a stream-recreate migration rather than an in-place change. - **Publish / deliver rates** are not exposed by JetStream stream info, so - `SourceStats` leaves those rate fields at zero. + `SourceStats` leaves those rate fields at zero. The message count for a + durable source is the consumer's backlog (undelivered plus unacked, with + `current_offset` from its ack floor) rather than the stream depth — the + stream retains acked messages under its retention limits, so its depth + keeps growing after consumers catch up, which would mislead anything using + message count as queue length (e.g. consumer autoscaling). Sources without + a durable consumer fall back to the stream view. - **Dead-letter is fire-and-forget.** Messages are republished to the dead-letter subject; there is no advisory-driven re-consumption. - **Connection authentication.** The connector supports user/password and TLS diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index c27edf5..418798d 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -711,6 +711,21 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb stats.MessageCount = int64(info.State.Msgs) //nolint:gosec stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec stats.LastOffset = int64(info.State.LastSeq) //nolint:gosec + // For a source with a durable consumer, report that consumer's actual + // backlog (undelivered + delivered-but-unacked) instead of the stream + // depth: the stream retains acked messages under its retention limits, so + // its message count keeps growing after consumers are caught up. This + // matches the amqp091 connector, which reports the queue's ready+unacked + // count — the number consumer-scaling logic wants. Without a durable + // consumer (or before it exists), fall back to the stream view. + if durable := durableName(source); durable != "" { + if cons, cerr := stream.Consumer(ctx, durable); cerr == nil { + if ci, ierr := cons.Info(ctx); ierr == nil { + stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec + stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec + } + } + } // Note: publish_rate / deliver_rate are not exposed directly by // JetStream stream info; they would need to be derived from sampling. Left // at zero for now. diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 8b8b63d..9f6db8c 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -579,3 +579,42 @@ func TestEmptyRoutingKeyDelivers(t *testing.T) { assert.Equal(t, "no-key", string(m.GetBody())) require.Nil(t, p.Ack(ctx, m.GetUuid())) } + +// TestSourceStatsReportsConsumerBacklog: for a durable source the message +// count is the consumer's backlog, not the stream depth — the stream retains +// acked messages under its retention limits, so its count never shrinks as +// consumers catch up. +func TestSourceStatsReportsConsumerBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.backlog", Subjects: []string{"e"}} + for i := 0; i < 3; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + } + + // Declare the durable consumer without consuming: its backlog is 3. + src := queueSource("events.backlog.consumer", "events.backlog", "e") + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Equal(t, int64(3), stats.GetMessageCount(), "durable backlog") + + // Drain and ack everything; backlog returns to 0 even though the stream + // still retains the acked messages. + src.DeclareOnly = false + out := make(chan *pb.Message, 3) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + for i := 0; i < 3; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + require.Eventually(t, func() bool { + st := p.SourceStats(ctx, src) + return st.GetError() == nil && st.GetMessageCount() == 0 + }, 10*time.Second, 200*time.Millisecond, "backlog drains to 0 after acks") +} From ed47b261d462734267c8276615bc795dd0f3af3a Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Fri, 3 Jul 2026 13:19:23 -0400 Subject: [PATCH 06/77] docs: state the subject scheme as a persistence contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The design doc framed the subject encoding as something with an upgrade path from an older scheme. No older scheme has ever shipped — it only existed in unmerged history on this branch — so drop the upgrade instructions and instead state the durable fact: the subject layout is part of the connector's persistence contract, any future encoding change is breaking for deployed data, and the scheme (including the '~' delimiter) is canonical for anything reading JetStream directly. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 1841a29..6d21778 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -68,12 +68,14 @@ subjects — so wildcard characters are sanitized to `_` like any other illegal character. In address names, empty tokens and tokens equal to `~` are replaced with `_` rather than dropped, so distinct names keep distinct roots. -Changing the subject scheme is a breaking change for data already stored -under an older scheme: `CreateOrUpdateStream` moves the stream's captured -subjects forward, after which messages persisted under old-style subjects no -longer match any consumer filter and age out via the retention limits. Drain -consumers (or recreate streams) when upgrading a deployment that has retained -backlog. +This subject scheme is part of the connector's persistence contract: +retained messages are stored under these subjects for the life of the +stream's retention limits, so any future change to the encoding is a +breaking change for deployed data (`CreateOrUpdateStream` moves the stream's +captured subjects forward, after which messages stored under the previous +encoding no longer match any consumer filter and age out). Treat this layout +as canonical: external tooling that reads or writes JetStream subjects +directly must use the same mapping, including the `~` delimiter. AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a From 0fe7e1cc6b836dcf2186f88d2b41866d9ae4f951 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Fri, 3 Jul 2026 19:18:54 -0400 Subject: [PATCH 07/77] feat: map the full RabbitMQ Streams offset vocabulary in natsjs The connector advertises the `Offset` source option in SupportedSourceOptions, and the amqp091 connector's toStreamOffset accepts first/continue/last/next plus an absolute numeric offset. natsjs previously mapped only `first` to DeliverAll and silently degraded everything else -- including `last` and numeric offsets -- to DeliverNew, so a consumer asking to replay from the last message or from a given offset received only new messages, with no error. deliverPolicyFor now mirrors toStreamOffset: first/continue -> DeliverAll, last -> DeliverLast, next/"" -> DeliverNew, and an absolute number -> DeliverByStartSequence with OptStartSeq. Matching is case-insensitive, and offset 0 maps to DeliverAll because JetStream sequences are 1-based. Numeric offsets are JetStream stream sequence numbers (as surfaced by SourceStats) and are not portable across brokers; this is now stated in the code and the design doc. Adds behavioral tests for retained-log replay, independent per-consumer offsets, and the last/next/numeric start positions. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 12 +- internal/provider/connectors/natsjs/natsjs.go | 43 ++++- .../provider/connectors/natsjs/natsjs_test.go | 157 +++++++++++++++++- 3 files changed, 194 insertions(+), 18 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 6d21778..fd01b73 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -96,9 +96,13 @@ The connector chooses the consumer kind from the source auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use for per-instance, transient subscriptions. -The `DeliverPolicy` is taken from the `Offset` option (`first` -> deliver all, -otherwise deliver new) and only applies on first creation; a reconnecting -durable resumes from its stored ack position. +The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 +connector's offset vocabulary so both accept the same values: `first`/`continue` +-> deliver all, `last` -> the final message, `next` (or unset) -> deliver new, +and an absolute number -> start at that stream sequence. It only applies on +first creation; a reconnecting durable resumes from its stored ack position. +Numeric offsets are JetStream stream sequence numbers (as surfaced by +`SourceStats`) and are not portable across brokers. ## Ack, retry, and dead-letter mapping @@ -231,7 +235,7 @@ every `Source.Options` key the connector reads: | `Expires` | string (ms) | Source expiry when unused (mapped to stream `MaxAge`). | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing key for dead-lettered messages. | -| `Offset` | string | Stream starting offset (`first` / `next`). | +| `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | | `ConsumerGroup` | string | Durable consumer group name (Streams only). | ## Known limitations diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 418798d..52a5dc2 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -489,10 +490,12 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if prefetch <= 0 { prefetch = 1 } + deliverPolicy, startSeq := deliverPolicyFor(source) consCfg := jetstream.ConsumerConfig{ FilterSubjects: filterSubjectsFor(source), AckPolicy: jetstream.AckExplicitPolicy, - DeliverPolicy: deliverPolicyFor(source), + DeliverPolicy: deliverPolicy, + OptStartSeq: startSeq, // 0 (ignored) unless DeliverByStartSequence AckWait: defaultAckWait, MaxAckPending: prefetch, } @@ -550,16 +553,38 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c return nil } -func deliverPolicyFor(source *pb.Source) jetstream.DeliverPolicy { - // Maps the Offset option. Transient consumers typically want "new" (only - // messages after subscribe); stream replay consumers want "all". - switch source.GetOptions()["Offset"] { - case "first": - return jetstream.DeliverAllPolicy +// deliverPolicyFor maps the RabbitMQ-Streams `Offset` option onto a JetStream +// DeliverPolicy, mirroring the amqp091 connector's toStreamOffset so both +// connectors accept the same offset vocabulary. The second return value is the +// start sequence, meaningful only when the policy is DeliverByStartSequence. +// +// Mapping: `first`/`continue` -> deliver all (a durable "continue"s from its +// stored ack floor on reconnect regardless, so all-from-start is the correct +// first-creation fallback); `last` -> the final message; `next`/"" -> only +// messages published after the consumer is created; an absolute number -> +// start at that stream sequence. +// +// NOTE: numeric offsets are JetStream stream sequence numbers (as reported by +// SourceStats), which are 1-based and not portable across brokers. Sequence 0 +// therefore means "from the beginning" and maps to DeliverAll rather than an +// invalid start sequence. An unparseable offset falls back to "new". +func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64) { + off := source.GetOptions()["Offset"] + switch strings.ToLower(off) { + case "first", "continue": + return jetstream.DeliverAllPolicy, 0 + case "last": + return jetstream.DeliverLastPolicy, 0 case "next", "": - return jetstream.DeliverNewPolicy + return jetstream.DeliverNewPolicy, 0 default: - return jetstream.DeliverNewPolicy + if seq, err := strconv.ParseUint(strings.TrimSpace(off), 10, 64); err == nil { + if seq == 0 { + return jetstream.DeliverAllPolicy, 0 + } + return jetstream.DeliverByStartSequencePolicy, seq + } + return jetstream.DeliverNewPolicy, 0 } } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 9f6db8c..1266e3b 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -438,11 +438,25 @@ func TestStreamRegistryFollowerHonorsContext(t *testing.T) { func TestDeliverPolicyFor(t *testing.T) { mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } - assert.Equal(t, jetstream.DeliverAllPolicy, deliverPolicyFor(mk("first"))) - assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk("next"))) - assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk(""))) - assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(mk("garbage"))) - assert.Equal(t, jetstream.DeliverNewPolicy, deliverPolicyFor(&pb.Source{})) + check := func(off string, wantPol jetstream.DeliverPolicy, wantSeq uint64) { + t.Helper() + pol, seq := deliverPolicyFor(mk(off)) + assert.Equal(t, wantPol, pol, "policy for Offset=%q", off) + assert.Equal(t, wantSeq, seq, "start seq for Offset=%q", off) + } + check("first", jetstream.DeliverAllPolicy, 0) + check("First", jetstream.DeliverAllPolicy, 0) // case-insensitive, like amqp091 + check("continue", jetstream.DeliverAllPolicy, 0) + check("last", jetstream.DeliverLastPolicy, 0) + check("next", jetstream.DeliverNewPolicy, 0) + check("", jetstream.DeliverNewPolicy, 0) + check("100", jetstream.DeliverByStartSequencePolicy, 100) + check("0", jetstream.DeliverAllPolicy, 0) // offset 0 == from the beginning + check("garbage", jetstream.DeliverNewPolicy, 0) + + pol, seq := deliverPolicyFor(&pb.Source{}) + assert.Equal(t, jetstream.DeliverNewPolicy, pol) + assert.Equal(t, uint64(0), seq) } func TestFirstSubject(t *testing.T) { @@ -560,6 +574,139 @@ func TestPrefixAddressesCoexist(t *testing.T) { assert.True(t, got["to-parent"] && got["still-to-parent"], "got %v", got) } +// streamSource builds a stream-style source positioned at the given RabbitMQ +// Streams offset ("first"/"next"/...). It is the offset-parameterized sibling +// of queueSource. +func streamSource(name, addr, offset string, subjects ...string) *pb.Source { + return &pb.Source{ + Name: name, + Type: pb.Source_QUEUE, + Address: &pb.Address{Name: addr, Subjects: subjects}, + Options: map[string]string{"Offset": offset}, + } +} + +// TestStreamOffsetReplay is the RabbitMQ-Streams parity test. A natsjs address +// is a retained JetStream log (LimitsPolicy, acked messages are NOT deleted), +// so it behaves like a RabbitMQ Stream rather than a classic/quorum work +// queue: independent consumers each read from their own offset, and a fully +// acked backlog is still replayable by a fresh consumer. +func TestStreamOffsetReplay(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.stream", Subjects: []string{"e"}} + const n = 4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + drain := func(t *testing.T, name string) int { + t.Helper() + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource(name, "events.stream", "first", "e"), out) + for i := 0; i < n; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + return n + } + + // Consumer A reads and acks the whole backlog from offset "first". + t.Run("first_reads_full_backlog", func(t *testing.T) { + assert.Equal(t, n, drain(t, "events.stream.a")) + }) + + // The defining Streams property: a second, independent consumer with its + // own durable/offset still replays all n messages even though consumer A + // already acked them. A RabbitMQ classic/quorum queue would have nothing + // left to deliver here — a Stream does. + t.Run("independent_consumer_replays_after_acks", func(t *testing.T) { + assert.Equal(t, n, drain(t, "events.stream.b"), + "an independent stream consumer replays the retained log") + }) +} + +// TestStreamOffsetNextSkipsBacklog: offset "next" (and the default) positions a +// new consumer at the tail — it sees only messages published after it was +// created, matching RabbitMQ Streams "next". +func TestStreamOffsetNextSkipsBacklog(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.tail", Subjects: []string{"e"}} + // Backlog published before the consumer exists. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("old")})) + + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource("events.tail.consumer", "events.tail", "next", "e"), out) + // Let the durable form at the tail before publishing "new", so DeliverNew's + // start point is unambiguously after "old". + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("new")})) + + m := recv(t, out) + assert.Equal(t, "new", string(m.GetBody()), "next offset skips the pre-existing backlog") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected extra delivery (backlog leaked past 'next'): %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } +} + +// TestStreamOffsetLastAndNumeric is the behavioral counterpart to +// TestDeliverPolicyFor: it drives the two offset positions that amqp091's +// toStreamOffset supports and that natsjs now honors — "last" (final message +// only) and an absolute numeric stream sequence. +func TestStreamOffsetLastAndNumeric(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.offset", Subjects: []string{"e"}} + const n = 4 // seq: m0=1, m1=2, m2=3, m3=4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + t.Run("last_delivers_only_final", func(t *testing.T) { + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, streamSource("events.offset.last", "events.offset", "last", "e"), out) + + m := recv(t, out) + assert.Equal(t, "m3", string(m.GetBody()), "last offset starts at the final message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("last should deliver only the final message, got extra %q", extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + }) + + t.Run("numeric_offset_starts_at_sequence", func(t *testing.T) { + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + // JetStream sequence 3 == m2 (1-based); expect m2 then m3. + go p.Subscribe(subCtx, streamSource("events.offset.num", "events.offset", "3", "e"), out) + + got := []string{} + for i := 0; i < 2; i++ { + m := recv(t, out) + got = append(got, string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.Equal(t, []string{"m2", "m3"}, got, "numeric offset starts at the given stream sequence") + }) +} + // TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no // routing key: they map to the bare ".~" subject, which both the stream // and a pattern-less consumer capture. From 83dcf163709ef2ee230c00b0ecffde129b65e5c1 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 4 Jul 2026 08:21:16 -0400 Subject: [PATCH 08/77] fix: reject invalid offsets and keep durable re-subscribes working Two gaps remained after natsjs adopted the full RabbitMQ Streams offset vocabulary. An unrecognized Offset still fell back silently to DeliverNew. The amqp091 connector's offset parsing rejects such values, and starting a consumer at a silently different position than the one it asked for loses or replays data, so deliverPolicyFor now returns an error and the subscribe fails with it. JetStream fixes a durable consumer's start position at creation and rejects DeliverPolicy/OptStartSeq updates (err 10012), so re-subscribing an existing durable with a different Offset failed the subscribe -- permanently, since retrying cannot heal a config conflict. The documented contract is that the offset applies on first creation only and a reconnecting durable resumes from its stored ack position, so Subscribe now implements exactly that: when CreateOrUpdateConsumer fails and the durable already exists, it attaches to the existing consumer unchanged and logs a warning naming the conflict. Repositioning requires a new durable (source or ConsumerGroup) name. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 13 +++- internal/provider/connectors/natsjs/natsjs.go | 39 +++++++--- .../provider/connectors/natsjs/natsjs_test.go | 74 ++++++++++++++++++- 3 files changed, 110 insertions(+), 16 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index fd01b73..f09f5ab 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -99,10 +99,15 @@ The connector chooses the consumer kind from the source The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 connector's offset vocabulary so both accept the same values: `first`/`continue` -> deliver all, `last` -> the final message, `next` (or unset) -> deliver new, -and an absolute number -> start at that stream sequence. It only applies on -first creation; a reconnecting durable resumes from its stored ack position. -Numeric offsets are JetStream stream sequence numbers (as surfaced by -`SourceStats`) and are not portable across brokers. +and an absolute number -> start at that stream sequence. Any other value fails +the subscribe (as in amqp091's offset parsing) rather than silently starting +the consumer at a different position than it asked for. The offset only +applies on first creation: JetStream fixes a durable's start position when the +consumer is created, so a re-subscribe that requests a different offset logs a +warning and resumes from the durable's stored ack position — use a new durable +(source or `ConsumerGroup` name) to reposition. Numeric offsets are JetStream +stream sequence numbers (as surfaced by `SourceStats`) and are not portable +across brokers. ## Ack, retry, and dead-letter mapping diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 52a5dc2..5e2b8e2 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -490,7 +490,10 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if prefetch <= 0 { prefetch = 1 } - deliverPolicy, startSeq := deliverPolicyFor(source) + deliverPolicy, startSeq, derr := deliverPolicyFor(source) + if derr != nil { + return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), derr.Error()), IsFatal: true} + } consCfg := jetstream.ConsumerConfig{ FilterSubjects: filterSubjectsFor(source), AckPolicy: jetstream.AckExplicitPolicy, @@ -512,6 +515,21 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) + if cerr != nil && consCfg.Durable != "" { + // A durable's start position (DeliverPolicy/OptStartSeq) is fixed at + // creation — JetStream rejects updates to it (err 10012), so a client + // re-subscribing with a different Offset would otherwise fail here + // forever. The documented contract is that Offset applies on first + // creation only and a reconnecting durable resumes from its stored ack + // position, so attach to the existing consumer as-is; to reposition, + // use a new durable (source/ConsumerGroup) name. + if existing, aerr := stream.Consumer(tctx, consCfg.Durable); aerr == nil { + util.Logger.Warn( + "natsjs: durable consumer {0} exists with a different configuration ({1}); resuming it unchanged", + consCfg.Durable, cerr.Error()) + cons, cerr = existing, nil + } + } if cerr != nil { // Include the mapped filter subjects: a NATS "invalid subject" (10052) // otherwise gives no hint which source/subject produced it. @@ -567,24 +585,27 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // NOTE: numeric offsets are JetStream stream sequence numbers (as reported by // SourceStats), which are 1-based and not portable across brokers. Sequence 0 // therefore means "from the beginning" and maps to DeliverAll rather than an -// invalid start sequence. An unparseable offset falls back to "new". -func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64) { +// invalid start sequence. An unrecognized offset is an error, like +// toStreamOffset: starting a consumer at a silently different position than +// the one it asked for loses or replays data. +func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64, error) { off := source.GetOptions()["Offset"] switch strings.ToLower(off) { case "first", "continue": - return jetstream.DeliverAllPolicy, 0 + return jetstream.DeliverAllPolicy, 0, nil case "last": - return jetstream.DeliverLastPolicy, 0 + return jetstream.DeliverLastPolicy, 0, nil case "next", "": - return jetstream.DeliverNewPolicy, 0 + return jetstream.DeliverNewPolicy, 0, nil default: if seq, err := strconv.ParseUint(strings.TrimSpace(off), 10, 64); err == nil { if seq == 0 { - return jetstream.DeliverAllPolicy, 0 + return jetstream.DeliverAllPolicy, 0, nil } - return jetstream.DeliverByStartSequencePolicy, seq + return jetstream.DeliverByStartSequencePolicy, seq, nil } - return jetstream.DeliverNewPolicy, 0 + return jetstream.DeliverNewPolicy, 0, fmt.Errorf( + "invalid offset: %q (expected first, continue, last, next, or a numeric stream sequence)", off) } } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 1266e3b..761ab6c 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -440,7 +440,8 @@ func TestDeliverPolicyFor(t *testing.T) { mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } check := func(off string, wantPol jetstream.DeliverPolicy, wantSeq uint64) { t.Helper() - pol, seq := deliverPolicyFor(mk(off)) + pol, seq, err := deliverPolicyFor(mk(off)) + assert.Nil(t, err, "error for Offset=%q", off) assert.Equal(t, wantPol, pol, "policy for Offset=%q", off) assert.Equal(t, wantSeq, seq, "start seq for Offset=%q", off) } @@ -452,11 +453,18 @@ func TestDeliverPolicyFor(t *testing.T) { check("", jetstream.DeliverNewPolicy, 0) check("100", jetstream.DeliverByStartSequencePolicy, 100) check("0", jetstream.DeliverAllPolicy, 0) // offset 0 == from the beginning - check("garbage", jetstream.DeliverNewPolicy, 0) - pol, seq := deliverPolicyFor(&pb.Source{}) + pol, seq, err := deliverPolicyFor(&pb.Source{}) + assert.Nil(t, err) assert.Equal(t, jetstream.DeliverNewPolicy, pol) assert.Equal(t, uint64(0), seq) + + // An unrecognized offset is rejected, like amqp091's toStreamOffset — + // starting at a silently different position would lose or replay data. + for _, bad := range []string{"garbage", "-1", "1.5"} { + _, _, err := deliverPolicyFor(mk(bad)) + assert.ErrorContains(t, err, "invalid offset", "Offset=%q", bad) + } } func TestFirstSubject(t *testing.T) { @@ -707,6 +715,66 @@ func TestStreamOffsetLastAndNumeric(t *testing.T) { }) } +// TestSubscribeInvalidOffsetRejected: an offset outside the shared vocabulary +// fails the subscribe (mirroring amqp091's toStreamOffset) instead of silently +// starting the consumer at "next". +func TestSubscribeInvalidOffsetRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + perr := p.Subscribe(ctx, streamSource("events.bad.consumer", "events.bad", "latest", "e"), out) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "invalid offset") + assert.True(t, perr.GetIsFatal()) +} + +// TestDurableOffsetPinnedAtCreation: a durable's start position is fixed when +// the consumer is first created — JetStream rejects DeliverPolicy/OptStartSeq +// updates (err 10012) — so a re-subscribe that asks for a different Offset must +// not fail; it attaches to the existing durable and resumes from its stored ack +// position, per the documented contract. +func TestDurableOffsetPinnedAtCreation(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.pinned", Subjects: []string{"e"}} + const n = 4 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + src := func(offset string) *pb.Source { + return streamSource("events.pinned.consumer", "events.pinned", offset, "e") + } + + // Create the durable at "first" and ack only m0, leaving m1..m3 pending. + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, src("first"), out) }() + m := recv(t, out) + assert.Equal(t, "m0", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe the same durable asking for sequence 4 (m3). The request + // conflicts with the pinned start position; the consumer resumes from its + // stored ack floor instead, so the next delivery is m1, not m3. + out2 := make(chan *pb.Message, n) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + errCh2 := make(chan *pb.Error, 1) + go func() { errCh2 <- p.Subscribe(subCtx2, src("4"), out2) }() + m2 := recv(t, out2) + assert.Equal(t, "m1", string(m2.GetBody()), + "existing durable resumes from its stored position; the conflicting offset is ignored") + require.Nil(t, p.Ack(ctx, m2.GetUuid())) + cancel2() + require.Nil(t, <-errCh2, "conflicting offset on an existing durable must not fail the subscribe") +} + // TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no // routing key: they map to the bare ".~" subject, which both the stream // and a pattern-less consumer capture. From 7cb42b323ab0cc6a0a7cfbcce06def7c72bd2abe Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 4 Jul 2026 08:21:43 -0400 Subject: [PATCH 09/77] docs: describe MessageTTL and Expires as accepted but not applied The source-options table claimed both options were mapped to the stream MaxAge. Neither is read by the connector: retention comes solely from the stream-wide NATSJS_STREAM_MAX_AGE / NATSJS_STREAM_MAX_BYTES configuration, because a per-source value cannot be mapped onto the shared per-address-root stream without one source's value flapping another's retention. The options stay in SupportedSourceOptions so existing amqp091 client sources validate unchanged; the doc now states what actually happens instead of implying per-source fidelity. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index f09f5ab..83953dd 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -232,12 +232,13 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; ## Source options Per the connector-interface contract, `SupportedSourceOptions()` advertises -every `Source.Options` key the connector reads: +the `Source.Options` keys the connector accepts — the same list as the +amqp091 connector, so existing client sources validate unchanged: | Key | Type | Description | | --- | --- | --- | -| `MessageTTL` | string (ms) | Per-source message TTL (mapped to stream `MaxAge`). | -| `Expires` | string (ms) | Source expiry when unused (mapped to stream `MaxAge`). | +| `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). | +| `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing key for dead-lettered messages. | | `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | @@ -246,12 +247,14 @@ every `Source.Options` key the connector reads: ## Known limitations - **Per-source TTL fidelity.** The connector uses one stream per address - root, so `MessageTTL` / `Expires` can only be honored as a stream-wide - `MaxAge`, not a faithful per-source TTL. True per-source TTL (or switching - queue sources to a delete-on-ack policy such as `WorkQueuePolicy` / - `InterestPolicy`) needs a per-source stream topology, and `Retention` is - immutable, so that is a stream-recreate migration rather than an in-place - change. + root, so a per-source `MessageTTL` / `Expires` cannot be mapped onto the + shared stream without one source's value flapping another's retention. + Both options are therefore accepted but ignored; retention comes from the + stream-wide `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` + configuration. True per-source TTL (or switching queue sources to a + delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a + per-source stream topology, and `Retention` is immutable, so that is a + stream-recreate migration rather than an in-place change. - **Publish / deliver rates** are not exposed by JetStream stream info, so `SourceStats` leaves those rate fields at zero. The message count for a durable source is the consumer's backlog (undelivered plus unacked, with From eb53b80d6297b3905e5b2204c1a7bdf4a805eb20 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 4 Jul 2026 08:21:58 -0400 Subject: [PATCH 10/77] chore(deps): bump embedded nats-server test dep to v2.14.3 The 2.11 line reached end of life with 2.11.17; patch releases now ship on the 2.12.x and 2.14.x lines. The connector's behavioral suite runs against the embedded server, so testing against a current release matches what new deployments run. No connector changes were needed -- it uses only the nats.go jetstream package, and nats.go stays at 1.52.0. Transitive bumps: jwt v2.8.2, nkeys v0.4.16, x/crypto v0.53.0, x/sys v0.46.0, x/term v0.44.0. Signed-off-by: Michael Otteni --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 733a7bf..0377a82 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/hashicorp/go-metrics v0.5.4 - github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats-server/v2 v2.14.3 github.com/nats-io/nats.go v1.52.0 github.com/prometheus/client_golang v1.23.2 github.com/rabbitmq/amqp091-go v1.11.0 @@ -59,8 +59,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect @@ -79,11 +79,11 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/go.sum b/go.sum index f984c2f..72af085 100644 --- a/go.sum +++ b/go.sum @@ -152,14 +152,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= -github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= -github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= -github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.14.3 h1:+xjydPt7rkit67G+04TN0mcO2n+8nveZE7tK/PPV53A= +github.com/nats-io/nats-server/v2 v2.14.3/go.mod h1:5IlCtBzfwyzQzPMjmoJ9W2/LKmnJRtNyuOs/OT+NHDY= github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= @@ -270,8 +270,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= @@ -314,10 +314,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 653c4220d767eaccdaa59445f89b92033bee19da Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 12:21:33 -0400 Subject: [PATCH 11/77] fix: delete ephemeral consumers when their subscription ends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ephemeral consumers were left to expire via InactiveThreshold (5m). Each one counts against the server's per-stream consumer limit until then, so clients churning transient subscriptions faster than the threshold would accumulate dead consumers on the server. Delete the consumer eagerly when the subscription tears down; the threshold remains as the janitor for unclean exits. The delete is best-effort on a detached, bounded context — the subscription's own context is already cancelled when it runs. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 5 ++- internal/provider/connectors/natsjs/natsjs.go | 25 +++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 45 +++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 83953dd..fa65a79 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -94,7 +94,10 @@ The connector chooses the consumer kind from the source and any `SingleActiveConsumer`. - Ephemeral consumers auto-expire after an inactivity threshold. Used for auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use - for per-instance, transient subscriptions. + for per-instance, transient subscriptions. They are also deleted eagerly + when their subscription ends, so the threshold only has to cover unclean + exits and churning transient clients cannot accumulate dead consumers + against the server's per-stream consumer limit. The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 connector's offset vocabulary so both accept the same values: `first`/`continue` diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 5e2b8e2..a058d94 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -55,6 +55,10 @@ const ( // consume error handler and the client re-issues its pull request after // roughly twice this interval, instead of ~30s with the library defaults. defaultConsumeHeartbeat = 5 * time.Second + // ephemeralDeleteTimeout bounds the best-effort DeleteConsumer issued when + // an ephemeral subscription ends; teardown must not hang on a broker that + // is already gone. + ephemeralDeleteTimeout = 5 * time.Second // defaultStreamMaxAge bounds how long a stream retains messages so the // JetStream log does not grow without bound (RabbitMQ deletes on ack; // JetStream LimitsPolicy does not). 72h is generous enough not to truncate a @@ -565,12 +569,33 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c defer func() { cc.Stop() bd.consumeContexts.Delete(source.GetName()) + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + } }() <-ctx.Done() return nil } +// deleteEphemeralConsumer removes an ephemeral consumer as soon as its +// subscription ends. Ephemeral consumers do expire on their own after +// InactiveThreshold, but until then each one still counts against the server's +// per-stream consumer limit, so clients that churn transient subscriptions +// faster than the threshold would accumulate dead consumers on the server. +// Deleting eagerly leaves the threshold as a janitor for unclean exits (a +// crashed client never reaches this path) only. Best-effort: the subscription +// context is already cancelled when this runs, so the call gets a detached, +// bounded context, and a failure is only logged — the consumer then simply +// expires via the threshold as before. +func deleteEphemeralConsumer(ctx context.Context, stream jetstream.Stream, name string) { + dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), ephemeralDeleteTimeout) + defer cancel() + if err := stream.DeleteConsumer(dctx, name); err != nil { + util.Logger.Debugf("natsjs: delete ephemeral consumer %s: %s", name, err.Error()) + } +} + // deliverPolicyFor maps the RabbitMQ-Streams `Offset` option onto a JetStream // DeliverPolicy, mirroring the amqp091 connector's toStreamOffset so both // connectors accept the same offset vocabulary. The second return value is the diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 761ab6c..2cd9b52 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -833,3 +833,48 @@ func TestSourceStatsReportsConsumerBacklog(t *testing.T) { return st.GetError() == nil && st.GetMessageCount() == 0 }, 10*time.Second, 200*time.Millisecond, "backlog drains to 0 after acks") } + +// TestEphemeralConsumerDeletedOnTeardown verifies a transient source's +// ephemeral consumer is removed from the server as soon as its subscription +// ends, instead of counting against the stream's consumer limit until the +// inactivity threshold expires it. +func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.tmp.listener", + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + Address: &pb.Address{Name: "events.tmp", Subjects: []string{"a"}}, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + _ = p.Subscribe(subCtx, src, out) + close(done) + }() + + bd := p.getBrokerDetailsByIdentifier("test-client") + var stream jetstream.Stream + require.Eventually(t, func() bool { + st, err := bd.js.Stream(ctx, streamNameFor("events.tmp")) + if err != nil { + return false + } + stream = st + info, ierr := st.Info(ctx) + return ierr == nil && info.State.Consumers == 1 + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer never appeared") + + cancel() + <-done + + assert.Eventually(t, func() bool { + info, err := stream.Info(ctx) + return err == nil && info.State.Consumers == 0 + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer was not deleted on teardown") +} From dbcb19b147e9ba5296245a167ea95fe8a5eca162 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 12:23:16 -0400 Subject: [PATCH 12/77] feat: sample publish and deliver rates for SourceStats The amqp091 connector fills SourceStats publish_rate/deliver_rate from the RabbitMQ management API; JetStream exposes only absolute counters, so those fields were left at zero. Derive them proxy-side instead: a per-connection rateTracker differences the stream's LastSeq (publish) and the durable consumer's delivered count (deliver) between successive SourceStats calls. The first observation baselines at zero, a counter that moves backwards (recreated stream or consumer) re-baselines, and a non-advancing clock reports zero rather than dividing by zero. Caveats, also documented: the publish rate covers the whole address stream rather than a single binding, and sources without a durable consumer have no consumer identity to sample, so their deliver rate stays zero. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 22 ++++---- internal/provider/connectors/natsjs/natsjs.go | 48 +++++++++++++++-- .../provider/connectors/natsjs/natsjs_test.go | 53 +++++++++++++++++++ 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index fa65a79..bc6d5e9 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -229,7 +229,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters`. | | `MessageTTL` / `Expires` | Partial | Mapped onto stream-level `MaxAge` (see limitations). | | Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | -| Publish / deliver rates in stats | Drop | Not exposed by JetStream info; would need sampling. | +| Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | | RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | ## Source options @@ -258,14 +258,18 @@ amqp091 connector, so existing client sources validate unchanged: delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a per-source stream topology, and `Retention` is immutable, so that is a stream-recreate migration rather than an in-place change. -- **Publish / deliver rates** are not exposed by JetStream stream info, so - `SourceStats` leaves those rate fields at zero. The message count for a - durable source is the consumer's backlog (undelivered plus unacked, with - `current_offset` from its ack floor) rather than the stream depth — the - stream retains acked messages under its retention limits, so its depth - keeps growing after consumers catch up, which would mislead anything using - message count as queue length (e.g. consumer autoscaling). Sources without - a durable consumer fall back to the stream view. +- **Publish / deliver rates are sampled, not native.** JetStream exposes + absolute counters, not rates, so `SourceStats` differences them between + successive calls: the first call after a (re)connect returns zero, the + publish rate covers the whole address stream rather than a single binding, + and sources without a durable consumer report a zero deliver rate. The + message count for a durable source is the consumer's backlog (undelivered + plus unacked, with `current_offset` from its ack floor) rather than the + stream depth — the stream retains acked messages under its retention + limits, so its depth keeps growing after consumers catch up, which would + mislead anything using message count as queue length (e.g. consumer + autoscaling). Sources without a durable consumer fall back to the stream + view. - **Dead-letter is fire-and-forget.** Messages are republished to the dead-letter subject; there is no advisory-driven re-consumption. - **Connection authentication.** The connector supports user/password and TLS diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index a058d94..a4f2b55 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -92,6 +92,8 @@ type natsBrokerDetails struct { knownStreams *util.ConcurrentMap // consumeContexts maps source name -> jetstream.ConsumeContext for teardown. consumeContexts *util.ConcurrentMap + // rates derives SourceStats publish/deliver rates from JetStream counters. + rates *rateTracker state atomic.Uint32 consumed int64 @@ -273,6 +275,7 @@ func (p *natsjsProvider) Connect(ctx context.Context, cfg *pb.ConnectionConfigur activeMessages: util.NewConcurrentMap(), knownStreams: util.NewConcurrentMap(), consumeContexts: util.NewConcurrentMap(), + rates: newRateTracker(), } bd.state.Store(provider.CONNECTING) @@ -761,6 +764,39 @@ func (p *natsjsProvider) Stats() *provider.Stats { return stats } +// rateTracker derives messages-per-second rates from the monotonic counters +// JetStream exposes. RabbitMQ's management API computes publish/deliver rates +// server-side; JetStream reports only absolute counters, so the connector +// differences them between successive SourceStats calls. The first +// observation of a key establishes a baseline and reports zero, as does a +// counter that moved backwards (a recreated stream or consumer). +type rateTracker struct { + mu sync.Mutex + samples map[string]rateSample +} + +type rateSample struct { + when time.Time + count uint64 +} + +func newRateTracker() *rateTracker { + return &rateTracker{samples: make(map[string]rateSample)} +} + +// observe records (now, count) for key and returns the average rate since the +// previous observation of that key. +func (rt *rateTracker) observe(key string, now time.Time, count uint64) float32 { + rt.mu.Lock() + defer rt.mu.Unlock() + prev, ok := rt.samples[key] + rt.samples[key] = rateSample{when: now, count: count} + if !ok || count < prev.count || !now.After(prev.when) { + return 0 + } + return float32(float64(count-prev.count) / now.Sub(prev.when).Seconds()) +} + func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb.SourceStats { stats := &pb.SourceStats{Name: source.GetName()} bd, err := p.getBrokerDetails(ctx) @@ -779,9 +815,14 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb stats.Error = &pb.Error{Message: ierr.Error()} return stats } + now := time.Now() stats.MessageCount = int64(info.State.Msgs) //nolint:gosec stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec stats.LastOffset = int64(info.State.LastSeq) //nolint:gosec + // JetStream exposes no rates, only counters; sample them between calls + // (see rateTracker). The stream's LastSeq counts every publish to the + // address root, so the publish rate is per address, not per binding. + stats.PublishRate = bd.rates.observe("pub/"+streamName, now, info.State.LastSeq) // For a source with a durable consumer, report that consumer's actual // backlog (undelivered + delivered-but-unacked) instead of the stream // depth: the stream retains acked messages under its retention limits, so @@ -794,11 +835,12 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb if ci, ierr := cons.Info(ctx); ierr == nil { stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec + // Delivered.Consumer counts deliveries (redeliveries included, + // like RabbitMQ's deliver rate). Ephemeral sources have no + // consumer identity to sample here, so their rate stays zero. + stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable, now, ci.Delivered.Consumer) } } } - // Note: publish_rate / deliver_rate are not exposed directly by - // JetStream stream info; they would need to be derived from sampling. Left - // at zero for now. return stats } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 2cd9b52..5eab99f 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -878,3 +878,56 @@ func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { return err == nil && info.State.Consumers == 0 }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer was not deleted on teardown") } + +func TestRateTracker(t *testing.T) { + rt := newRateTracker() + t0 := time.Now() + + // First observation only establishes the baseline. + assert.Zero(t, rt.observe("k", t0, 100)) + // 10 messages over 2 seconds. + assert.InDelta(t, 5.0, rt.observe("k", t0.Add(2*time.Second), 110), 1e-6) + // A counter that moved backwards (stream recreated) re-baselines at zero. + assert.Zero(t, rt.observe("k", t0.Add(3*time.Second), 4)) + assert.InDelta(t, 6.0, rt.observe("k", t0.Add(4*time.Second), 10), 1e-6) + // A non-advancing clock cannot divide by zero. + assert.Zero(t, rt.observe("k", t0.Add(4*time.Second), 20)) + // Keys are independent. + assert.Zero(t, rt.observe("other", t0, 1)) +} + +// TestSourceStatsRates verifies publish/deliver rates are sampled between +// SourceStats calls: the first call baselines at zero, and a later call after +// publishing and consuming reports positive rates. +func TestSourceStatsRates(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.rates", Subjects: []string{"e"}} + src := queueSource("events.rates.consumer", "events.rates", "e") + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + require.Nil(t, p.Ack(ctx, recv(t, out).GetUuid())) + + // Baseline scrape: no previous observation, rates are zero. + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Zero(t, stats.GetPublishRate()) + assert.Zero(t, stats.GetDeliverRate()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + require.Nil(t, p.Ack(ctx, recv(t, out).GetUuid())) + } + time.Sleep(50 * time.Millisecond) // guarantee a measurable interval + + stats = p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), "5 publishes since the last scrape") + assert.Positive(t, stats.GetDeliverRate(), "5 deliveries since the last scrape") +} From 617051cbe22c8744ca7e6b5281b492f62b6858e8 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 12:24:21 -0400 Subject: [PATCH 13/77] fix: warn when a source sets retention options natsjs does not apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MessageTTL and Expires stay in SupportedSourceOptions so sources written for the amqp091 connector validate unchanged, but natsjs retention is stream-wide and the options are not applied. That was documented but otherwise silent — and silent divergence in data retention is the one place a client should not have to read a design doc to notice. Subscribe now logs a warning naming the source and the ignored options. Rejecting such sources outright was considered and dropped: it would fail every existing client that legitimately sets a TTL the moment it is pointed at natsjs, turning a retention caveat into an outage. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 12 ++++++---- internal/provider/connectors/natsjs/natsjs.go | 23 +++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 10 ++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index bc6d5e9..b9ecefe 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -240,8 +240,8 @@ amqp091 connector, so existing client sources validate unchanged: | Key | Type | Description | | --- | --- | --- | -| `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). | -| `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. | +| `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). A warning is logged at subscribe time. | +| `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. A warning is logged at subscribe time. | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing key for dead-lettered messages. | | `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | @@ -252,9 +252,11 @@ amqp091 connector, so existing client sources validate unchanged: - **Per-source TTL fidelity.** The connector uses one stream per address root, so a per-source `MessageTTL` / `Expires` cannot be mapped onto the shared stream without one source's value flapping another's retention. - Both options are therefore accepted but ignored; retention comes from the - stream-wide `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` - configuration. True per-source TTL (or switching queue sources to a + Both options are therefore accepted but not applied, and a subscribe that + sets either logs a warning naming the source — silent divergence in data + retention is the one place a client must not have to read a design doc to + notice. Retention comes from the stream-wide `NATSJS_STREAM_MAX_AGE` / + `NATSJS_STREAM_MAX_BYTES` configuration. True per-source TTL (or switching queue sources to a delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a per-source stream topology, and `Retention` is immutable, so that is a stream-recreate migration rather than an in-place change. diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index a4f2b55..400f54d 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -477,6 +477,15 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if err != nil { return &pb.Error{Message: err.Error(), IsFatal: true} } + // MessageTTL/Expires are accepted so existing client sources validate + // against SupportedSourceOptions, but retention here is stream-wide: warn + // instead of silently ignoring them, so a source owner asking for a short + // TTL learns their data outlives it without reading the design doc. + if unapplied := unappliedSourceOptions(source); len(unapplied) > 0 { + util.Logger.Warn( + "natsjs: source {0} sets {1}, which natsjs accepts but does not apply; retention is stream-wide (NATSJS_STREAM_MAX_AGE / NATSJS_STREAM_MAX_BYTES)", + source.GetName(), strings.Join(unapplied, ", ")) + } addr := source.GetAddress().GetName() streamName, serr := p.ensureStream(ctx, bd, addr) if serr != nil { @@ -599,6 +608,20 @@ func deleteEphemeralConsumer(ctx context.Context, stream jetstream.Stream, name } } +// unappliedSourceOptions returns the retention options the source sets that +// natsjs accepts (so sources written for the amqp091 connector still validate) +// but does not apply — retention on natsjs is stream-wide (see streamMaxAge +// and the design doc's Known limitations). +func unappliedSourceOptions(source *pb.Source) []string { + var unapplied []string + for _, opt := range []string{"MessageTTL", "Expires"} { + if source.GetOptions()[opt] != "" { + unapplied = append(unapplied, opt) + } + } + return unapplied +} + // deliverPolicyFor maps the RabbitMQ-Streams `Offset` option onto a JetStream // DeliverPolicy, mirroring the amqp091 connector's toStreamOffset so both // connectors accept the same offset vocabulary. The second return value is the diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 5eab99f..434057b 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -931,3 +931,13 @@ func TestSourceStatsRates(t *testing.T) { assert.Positive(t, stats.GetPublishRate(), "5 publishes since the last scrape") assert.Positive(t, stats.GetDeliverRate(), "5 deliveries since the last scrape") } + +func TestUnappliedSourceOptions(t *testing.T) { + src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } + + assert.Empty(t, unappliedSourceOptions(src(nil))) + assert.Empty(t, unappliedSourceOptions(src(map[string]string{"Offset": "first", "MessageTTL": ""}))) + assert.Equal(t, []string{"MessageTTL"}, unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000"}))) + assert.Equal(t, []string{"MessageTTL", "Expires"}, + unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000", "Expires": "600000"}))) +} From 2c52b65045765947867ca5fd88e6e3ada2c183fe Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 12:24:53 -0400 Subject: [PATCH 14/77] docs: call out the cost profile of proxy-side header filtering Header filters cannot narrow server-side delivery on NATS, so the connector receives and discards non-matching messages. State plainly that the cost scales with subject-matched traffic, not header-matched traffic, and when to remodel header routing onto subjects. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index b9ecefe..43187ba 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -226,7 +226,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | | Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | | Dead-letter (DLX) | Proxy | No native DLX; republish to DLQ subject then `Term()`. | -| Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters`. | +| Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters` (see limitations). | | `MessageTTL` / `Expires` | Partial | Mapped onto stream-level `MaxAge` (see limitations). | | Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | | Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | @@ -256,8 +256,9 @@ amqp091 connector, so existing client sources validate unchanged: sets either logs a warning naming the source — silent divergence in data retention is the one place a client must not have to read a design doc to notice. Retention comes from the stream-wide `NATSJS_STREAM_MAX_AGE` / - `NATSJS_STREAM_MAX_BYTES` configuration. True per-source TTL (or switching queue sources to a - delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a + `NATSJS_STREAM_MAX_BYTES` configuration. True per-source TTL (or switching + queue sources to a delete-on-ack policy such as `WorkQueuePolicy` / + `InterestPolicy`) needs a per-source stream topology, and `Retention` is immutable, so that is a stream-recreate migration rather than an in-place change. - **Publish / deliver rates are sampled, not native.** JetStream exposes @@ -272,6 +273,15 @@ amqp091 connector, so existing client sources validate unchanged: mislead anything using message count as queue length (e.g. consumer autoscaling). Sources without a durable consumer fall back to the stream view. +- **Header filters are evaluated proxy-side.** NATS routes on subjects only, + so a source's header `Filter`s cannot narrow what the server delivers: + every message matching the source's subject filters is delivered to the + connector, which evaluates the headers and acks non-matching messages + without forwarding them. Bandwidth and CPU between broker and proxy + therefore scale with the subject-matched traffic, not the header-matched + traffic. That is fine when header filters refine an already-narrow subject, + but a high-volume address consumed almost entirely through header filters + should be remodeled onto routing keys (subjects) instead. - **Dead-letter is fire-and-forget.** Messages are republished to the dead-letter subject; there is no advisory-driven re-consumption. - **Connection authentication.** The connector supports user/password and TLS From a46a8e262528238e6753b68c7435ea82ae5f43a8 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 13:41:17 -0400 Subject: [PATCH 15/77] fix: keep a message in flight when its dead-letter publish fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeadLetter published to the DLQ best-effort and then Term'd the original unconditionally: a failed ensure of the DLQ stream was skipped silently and a failed DLQ publish was only logged at debug, yet the message was still removed from redelivery — gone from both the source and the DLQ. RabbitMQ has no such window because the DLX move happens broker-side. Order the two steps so the data survives: Term only after the DLQ publish succeeds. On failure, warn and return an error while leaving the message resolvable and ack-pending; the server then falls back to a nack, so the message is redelivered and dead-lettering is retried. The DLQ copy now carries a Nats-Msg-Id derived from the original's stream sequence so a retried dead-letter cannot duplicate it within the dedup window (the header map is copied to keep the option from mutating the original message). Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 16 +++++- .../provider/connectors/natsjs/helpers.go | 14 +++++ internal/provider/connectors/natsjs/natsjs.go | 56 +++++++++++++++---- .../provider/connectors/natsjs/natsjs_test.go | 50 +++++++++++++++++ 4 files changed, 121 insertions(+), 15 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 43187ba..6ae5eda 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -129,7 +129,15 @@ dead-letter retry-queue idiom; JetStream increments the delivery count, which the connector surfaces back to the client as the `x-retry-count` header (see below). JetStream has no native dead-letter exchange, so `DeadLetter` republishes the message to the configured dead-letter subject and then -`Term()`s it to stop redelivery. +`Term()`s it to stop redelivery. RabbitMQ performs that move broker-side; +here it is two proxy-side steps, so ordering is what protects the data: the +original is terminated only after the DLQ publish succeeds. If the DLQ stream +cannot be ensured or published to, `DeadLetter` returns an error and leaves +the message ack-pending — the server then falls back to a nack, so the +message is redelivered and dead-lettering is retried instead of the message +being lost. The DLQ copy carries a `Nats-Msg-Id` derived from the original's +stream sequence, so a retried dead-letter of the same message deduplicates in +the DLQ within its dedup window. ## Retry-count header @@ -282,8 +290,10 @@ amqp091 connector, so existing client sources validate unchanged: traffic. That is fine when header filters refine an already-narrow subject, but a high-volume address consumed almost entirely through header filters should be remodeled onto routing keys (subjects) instead. -- **Dead-letter is fire-and-forget.** Messages are republished to the - dead-letter subject; there is no advisory-driven re-consumption. +- **Dead-letter is a proxy-side republish.** The DLQ is an ordinary stream + fed by the connector; there is no advisory-driven re-consumption. A failed + DLQ publish fails the dead-letter call — the message stays in flight and is + redelivered — rather than dropping the message. - **Connection authentication.** The connector supports user/password and TLS today; NKEYs / JWT auth are a natural follow-up for production deployments. diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index c03f342..cb761c1 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -246,6 +246,20 @@ func pbToNatsHeader(in map[string]string) nats.Header { return h } +// copyHeader shallow-copies a NATS header so options applied to the copy at +// publish time (e.g. a message id) do not mutate the source message's header +// map. +func copyHeader(in nats.Header) nats.Header { + if len(in) == 0 { + return nil + } + out := make(nats.Header, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + // natsToPbHeader converts a NATS header to Arke's flat string headers, keeping // the first value of each key. func natsToPbHeader(in nats.Header) map[string]string { diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 400f54d..2a0ad2a 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -741,24 +741,56 @@ func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, d // DeadLetter publishes the message to the configured dead-letter subject (there // is no native DLX in JetStream) and terminates redelivery. +// +// RabbitMQ moves a nacked message to its DLX broker-side; here the move is two +// proxy-side steps, so ordering is what protects the data: the original is +// Term'd only after the DLQ publish succeeds. If the DLQ stream cannot be +// ensured or published to, the message is left in flight — still ack-pending on +// the server and still resolvable by uuid — and an error is returned, so the +// caller's fallback (the server nacks on a DeadLetter error) redelivers it and +// dead-lettering is retried, instead of the message being removed from +// redelivery while also absent from the DLQ. The DLQ publish carries a +// Nats-Msg-Id derived from the original's stream sequence, so a retried +// dead-letter of the same message cannot duplicate it in the DLQ within the +// dedup window. func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid string) *pb.Error { - m, bd, perr := p.takeMessage(ctx, uuid) - if perr != nil { - return perr + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} } + m := mu.(jetstream.Msg) opts := source.GetOptions() if dla := opts["DeadLetterAddress"]; dla != "" { - if _, err := p.ensureStream(ctx, bd, dla); err == nil { - dlqMsg := &nats.Msg{ - Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), - Data: m.Data(), - Header: m.Headers(), - } - if _, err := bd.js.PublishMsg(ctx, dlqMsg); err != nil { - util.Logger.Debugf("natsjs: dead-letter publish failed for %s: %s", uuid, err.Error()) - } + if _, serr := p.ensureStream(ctx, bd, dla); serr != nil { + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (ensure stream for {1}: {2}); message stays in flight", + uuid, dla, serr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter ensure stream: %s", serr.Error())} + } + dlqMsg := &nats.Msg{ + Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), + Data: m.Data(), + // Copied so the publish options below do not mutate the original's + // header map. + Header: copyHeader(m.Headers()), + } + var pubOpts []jetstream.PublishOpt + if md, merr := m.Metadata(); merr == nil { + pubOpts = append(pubOpts, + jetstream.WithMsgID(fmt.Sprintf("dlq-%s-%d", md.Stream, md.Sequence.Stream))) + } + if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (publish to {1}: {2}); message stays in flight", + uuid, dlqMsg.Subject, perr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter publish: %s", perr.Error())} } } + bd.activeMessages.Delete(uuid) if err := m.Term(); err != nil { return &pb.Error{Message: err.Error()} } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 434057b..227cb12 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -222,6 +222,56 @@ func TestDeadLetter(t *testing.T) { }) assert.Nil(t, dlqStats.GetError()) assert.Equal(t, int64(1), dlqStats.GetMessageCount()) + + // The copy carries a deterministic Nats-Msg-Id, so a retried dead-letter of + // the same message dedups in the DLQ instead of duplicating. + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.NotEmpty(t, raw.Header.Get("Nats-Msg-Id"), "DLQ copy must carry a dedup message id") +} + +// TestDeadLetterFailureKeepsMessage: when the dead-letter publish cannot happen +// (here the DLQ address's subject space is already claimed by a foreign +// stream, so ensuring its stream fails), DeadLetter must NOT Term the +// original. It returns an error and leaves the message in flight, so the +// caller's fallback nack still resolves the uuid and the message is +// redelivered — the failure mode is retry, not silent loss. +func TestDeadLetterFailureKeepsMessage(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Claim the DLQ subject space so ensureStream for the DLA fails with + // "subjects overlap with an existing stream". + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.CreateStream(ctx, jetstream.StreamConfig{ + Name: "squatter", + Subjects: []string{"events.dlx.~", "events.dlx.~.>"}, + }) + require.NoError(t, serr) + + addr := &pb.Address{Name: "events.dlfail", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlfail.consumer", "events.dlfail", "job") + src.Options["DeadLetterAddress"] = "events.dlx" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "DeadLetter must fail when the DLQ publish cannot happen") + assert.Contains(t, perr.GetMessage(), "dead-letter") + + // Not Term'd: the same uuid is still nackable, and the nack redelivers. + require.Nil(t, p.Nack(ctx, m.GetUuid()), "a failed dead-letter leaves the message nackable") + again := recv(t, out) + assert.Equal(t, "poison", string(again.GetBody())) + require.Nil(t, p.Ack(ctx, again.GetUuid())) } func TestDeduplication(t *testing.T) { From 9fcaa5b6014f693f7b9bd8f1de2789ef7904fade Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 13:41:50 -0400 Subject: [PATCH 16/77] fix: map prefetch 0 to an unlimited ack window like amqp091 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AMQP prefetch 0 means unlimited, and the amqp091 connector honors that by only calling SetPrefetch for positive values, leaving the channel default (unlimited). natsjs clamped prefetch <= 0 to MaxAckPending 1 — the opposite extreme: one in-flight message per consumer, a silent throughput cliff for any client relying on the AMQP convention. Map <= 0 to MaxAckPending -1, JetStream's explicit unlimited. The pinned-offset test now sets prefetch 1 explicitly, since it depends on one-at-a-time delivery to keep its backlog undelivered across a re-subscribe. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 2 +- internal/provider/connectors/natsjs/natsjs.go | 6 +++- .../provider/connectors/natsjs/natsjs_test.go | 35 ++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 6ae5eda..bff4499 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -229,7 +229,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | | Streams: offsets, start position | Native | JetStream is a log; `DeliverPolicy` maps `Offset`. | | Single active consumer | Native | Consumer config (needs a durable name). | -| Prefetch / QoS | Native | `MaxAckPending`. | +| Prefetch / QoS | Native | `MaxAckPending`; prefetch 0 (AMQP "unlimited") maps to unlimited (-1). | | HA / quorum queues | Native | JetStream R3 (Raft) via `NATSJS_STREAM_REPLICAS`. | | Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | | Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 2a0ad2a..bd7e0b4 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -502,9 +502,13 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c return &pb.Error{Message: fmt.Sprintf("open stream: %s", serr.Error()), IsFatal: true} } + // AMQP prefetch 0 means unlimited (amqp091 leaves the channel default in + // that case rather than calling SetPrefetch), so map it to JetStream's + // explicit unlimited (-1) — not to the server default of 1000, and + // certainly not to 1, which would silently serialize the consumer. prefetch := int(source.GetPrefetchCount()) if prefetch <= 0 { - prefetch = 1 + prefetch = -1 } deliverPolicy, startSeq, derr := deliverPolicyFor(source) if derr != nil { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 227cb12..188095f 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -274,6 +274,33 @@ func TestDeadLetterFailureKeepsMessage(t *testing.T) { require.Nil(t, p.Ack(ctx, again.GetUuid())) } +// TestPrefetchMapsToMaxAckPending: a positive PrefetchCount becomes the +// consumer's MaxAckPending. A prefetch of 0 is the AMQP "unlimited" +// convention — the amqp091 connector honors it by leaving the channel +// default, which is unlimited — so it maps to JetStream's unlimited (-1), +// not to a one-message-at-a-time clamp. +func TestPrefetchMapsToMaxAckPending(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + bd := p.getBrokerDetailsByIdentifier("test-client") + maxAckPending := func(name string, prefetch int32) int { + t.Helper() + src := queueSource(name, "events.prefetch", "e") + src.PrefetchCount = prefetch + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.prefetch")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(src)) + require.NoError(t, cerr) + return cons.CachedInfo().Config.MaxAckPending + } + + assert.Equal(t, 7, maxAckPending("events.prefetch.limited", 7)) + assert.Equal(t, -1, maxAckPending("events.prefetch.unlimited", 0)) +} + func TestDeduplication(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -795,7 +822,13 @@ func TestDurableOffsetPinnedAtCreation(t *testing.T) { } src := func(offset string) *pb.Source { - return streamSource("events.pinned.consumer", "events.pinned", offset, "e") + s := streamSource("events.pinned.consumer", "events.pinned", offset, "e") + // One message in flight at a time, so m1..m3 stay undelivered while + // the first subscription is up and are immediately deliverable to the + // re-attach below (unlimited prefetch would leave them ack-pending + // until AckWait). + s.PrefetchCount = 1 + return s } // Create the durable at "first" and ack only m0, leaving m1..m3 pending. From ed1672d3f0460e7319b562429a3a5233dd36db8c Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 13:42:54 -0400 Subject: [PATCH 17/77] fix: only absorb start-position conflicts when resuming a durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The attach-existing-durable fallback fired on any CreateOrUpdateConsumer error whenever the durable already existed. It was aimed at JetStream's refusal to move a pinned start position (err 10012, deliver policy / start sequence), but as written it also masked unrelated update failures: a consumer whose update was rejected for any other reason would be resumed unchanged with only a warning — silently consuming with a configuration (ack policy, filter subjects) the source never asked for. Classify the error instead: fall back only when the API error is the consumer-create code carrying one of the server's start-position refusals (deliver policy / start sequence / start time can not be updated); every other error stays fatal. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 ++- internal/provider/connectors/natsjs/natsjs.go | 40 ++++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 58 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index bff4499..bc6ed85 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -108,9 +108,12 @@ the consumer at a different position than it asked for. The offset only applies on first creation: JetStream fixes a durable's start position when the consumer is created, so a re-subscribe that requests a different offset logs a warning and resumes from the durable's stored ack position — use a new durable -(source or `ConsumerGroup` name) to reposition. Numeric offsets are JetStream -stream sequence numbers (as surfaced by `SourceStats`) and are not portable -across brokers. +(source or `ConsumerGroup` name) to reposition. Only that start-position +conflict is absorbed; any other error creating or updating the consumer fails +the subscribe, since resuming a consumer whose configuration silently differs +from the requested one would consume the wrong way with no signal. Numeric +offsets are JetStream stream sequence numbers (as surfaced by `SourceStats`) +and are not portable across brokers. ## Ack, retry, and dead-letter mapping diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index bd7e0b4..179dce2 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -16,6 +16,7 @@ package natsjs import ( "context" "crypto/tls" + "errors" "fmt" "os" "strconv" @@ -535,17 +536,21 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) - if cerr != nil && consCfg.Durable != "" { + if cerr != nil && consCfg.Durable != "" && isStartPositionConflict(cerr) { // A durable's start position (DeliverPolicy/OptStartSeq) is fixed at // creation — JetStream rejects updates to it (err 10012), so a client // re-subscribing with a different Offset would otherwise fail here // forever. The documented contract is that Offset applies on first // creation only and a reconnecting durable resumes from its stored ack // position, so attach to the existing consumer as-is; to reposition, - // use a new durable (source/ConsumerGroup) name. + // use a new durable (source/ConsumerGroup) name. Only start-position + // conflicts are absorbed this way: any other configuration error stays + // fatal, because attaching to a consumer whose effective config + // silently differs from the requested one (wrong ack policy, stale + // filter subjects) would consume the wrong way without any signal. if existing, aerr := stream.Consumer(tctx, consCfg.Durable); aerr == nil { util.Logger.Warn( - "natsjs: durable consumer {0} exists with a different configuration ({1}); resuming it unchanged", + "natsjs: durable consumer {0} exists with a different start position ({1}); resuming it unchanged", consCfg.Durable, cerr.Error()) cons, cerr = existing, nil } @@ -664,6 +669,35 @@ func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64, error } } +// startPositionConflictMessages are the server's refusals to change a durable +// consumer's start position, which JetStream fixes at creation. They arrive +// wrapped in the generic consumer-create API error (err 10012) whose +// description carries the specific refusal, so the description is what +// distinguishes an immutable start position from every other create or update +// failure sharing that code. +var startPositionConflictMessages = []string{ + "deliver policy can not be updated", + "start sequence can not be updated", + "start time can not be updated", +} + +// isStartPositionConflict reports whether err is JetStream refusing to move an +// existing durable consumer's start position (DeliverPolicy / OptStartSeq / +// OptStartTime) — the one configuration conflict Subscribe absorbs by +// attaching to the existing consumer unchanged. +func isStartPositionConflict(err error) bool { + var apiErr *jetstream.APIError + if !errors.As(err, &apiErr) || apiErr.ErrorCode != jetstream.JSErrCodeConsumerCreate { + return false + } + for _, m := range startPositionConflictMessages { + if strings.Contains(apiErr.Description, m) { + return true + } + } + return false +} + func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, m jetstream.Msg, out chan<- *pb.Message) { defer util.RecoverPanic() diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 188095f..bf4f8f9 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -858,6 +858,64 @@ func TestDurableOffsetPinnedAtCreation(t *testing.T) { require.Nil(t, <-errCh2, "conflicting offset on an existing durable must not fail the subscribe") } +// TestIsStartPositionConflict pins down which consumer-create errors the +// durable fallback absorbs: only the server's refusals to move a start +// position. Everything else — other immutable-field refusals sharing err code +// 10012, other API errors, non-API errors — stays fatal. +func TestIsStartPositionConflict(t *testing.T) { + mk := func(code jetstream.ErrorCode, desc string) error { + return &jetstream.APIError{ErrorCode: code, Description: desc} + } + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "deliver policy can not be updated"))) + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "start sequence can not be updated"))) + assert.True(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "start time can not be updated"))) + + // Same error code, different refusal: not a start-position conflict. + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "replay policy can not be updated"))) + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeConsumerCreate, "invalid subject"))) + // Different code, or not an API error at all. + assert.False(t, isStartPositionConflict(mk(jetstream.JSErrCodeStreamNotFound, "stream not found"))) + assert.False(t, isStartPositionConflict(fmt.Errorf("deliver policy can not be updated"))) +} + +// TestDurableConflictBeyondStartPositionStaysFatal: the attach-unchanged +// fallback exists for start-position conflicts only. If the existing durable +// differs in any other way — here an incompatible ReplayPolicy — resuming it +// unchanged would silently consume with a configuration the source never +// asked for, so the subscribe must fail instead. +func TestDurableConflictBeyondStartPositionStaysFatal(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.replay", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + + src := queueSource("events.replay.consumer", "events.replay", "e") + + // A consumer with the connector's durable name already exists, matching on + // start position but differing in another immutable field. + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.replay")) + require.NoError(t, serr) + _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: durableName(src), + FilterSubjects: filterSubjectsFor(src), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, // matches Offset "first" + ReplayPolicy: jetstream.ReplayOriginalPolicy, + }) + require.NoError(t, cerr) + + // Bound the subscribe so a regression (wrongly attaching and blocking) + // fails the test instead of hanging it. + subCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + perr := p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) + require.NotNil(t, perr, "a non-start-position config conflict must fail the subscribe") + assert.True(t, perr.GetIsFatal()) + assert.Contains(t, perr.GetMessage(), "create consumer") +} + // TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no // routing key: they map to the bare ".~" subject, which both the stream // and a pattern-less consumer capture. From 7ffb60c05406e7908ba303aaeec49e1e33827e56 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 13:43:24 -0400 Subject: [PATCH 18/77] fix: clean up ephemeral consumers on declare-only and failed-consume exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eager delete of an ephemeral consumer ran only on the normal subscription teardown. A declare-only subscribe returned before registering that cleanup, and a Consume() failure returned past it, so in both cases the consumer lingered on the server for the full inactivity threshold. The declare-only case can never be consumed at all — its server-generated name is not surfaced — so delete it before returning, and do the same when consuming fails to start. Also log (at debug) when the ack of a header-filtered message fails: the redelivery is harmless, but it inflates NumDelivered and thus the synthesized x-retry-count of a message the client never saw, which is worth a trace when debugging retry counts. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 3 ++- internal/provider/connectors/natsjs/natsjs.go | 21 ++++++++++++--- .../provider/connectors/natsjs/natsjs_test.go | 27 +++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index bc6ed85..831d24b 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -95,7 +95,8 @@ The connector chooses the consumer kind from the source - Ephemeral consumers auto-expire after an inactivity threshold. Used for auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use for per-instance, transient subscriptions. They are also deleted eagerly - when their subscription ends, so the threshold only has to cover unclean + when their subscription ends (or never starts — a declare-only call, or a + failure to begin consuming), so the threshold only has to cover unclean exits and churning transient clients cannot accumulate dead consumers against the server's per-stream consumer limit. diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 179dce2..4f2e905 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -564,8 +564,14 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } } - // DeclareOnly: topology established, do not consume. + // DeclareOnly: topology established, do not consume. An ephemeral consumer + // created only to validate that topology is garbage the moment we return — + // its server-generated name is never seen again — so delete it eagerly + // rather than letting it linger for the full inactivity threshold. if source.GetDeclareOnly() { + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + } return nil } @@ -584,6 +590,9 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c }), ) if cerr != nil { + if consCfg.Durable == "" { + deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + } return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} } bd.consumeContexts.Add(source.GetName(), cc) @@ -708,9 +717,15 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai headers[retryCountHeaderName] = strconv.FormatUint(md.NumDelivered-1, 10) } - // Proxy-side replacement for RabbitMQ headers-exchange routing. + // Proxy-side replacement for RabbitMQ headers-exchange routing. A failed + // ack here only means the message will be redelivered and re-filtered — + // harmless, but it inflates NumDelivered (and so the synthesized + // x-retry-count) of a message the client never saw, so leave a trace. if !evaluateFilters(source.GetFilters(), headers) { - _ = m.Ack() + if err := m.Ack(); err != nil { + util.Logger.Debugf("natsjs: ack of header-filtered message on source %s failed: %s", + source.GetName(), err.Error()) + } return } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index bf4f8f9..be134d3 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1020,6 +1020,33 @@ func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer was not deleted on teardown") } +// TestDeclareOnlyEphemeralConsumerCleanedUp: DeclareOnly on a transient source +// validates topology, but the ephemeral consumer it creates can never be +// attached to afterwards (its server-generated name is not surfaced), so it is +// deleted before Subscribe returns instead of lingering until the inactivity +// threshold. +func TestDeclareOnlyEphemeralConsumerCleanedUp(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.tmpdecl.listener", + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + DeclareOnly: true, + Address: &pb.Address{Name: "events.tmpdecl", Subjects: []string{"a"}}, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + require.Nil(t, p.Subscribe(ctx, src, nil)) + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.tmpdecl")) + require.NoError(t, serr) + info, ierr := stream.Info(ctx) + require.NoError(t, ierr) + assert.Zero(t, info.State.Consumers, "declare-only ephemeral consumer must not linger") +} + func TestRateTracker(t *testing.T) { rt := newRateTracker() t0 := time.Now() From 06597f0b2b9bbae1002f5f23bfc37e0d979d225e Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 14:19:04 -0400 Subject: [PATCH 19/77] fix: release a subscription's in-flight messages when it ends A subscription's delivered-but-unacked messages could never be resolved after its consume stream closed (acks travel on that stream), yet their entries stayed in the active-message map for the life of the connection - a leak that also inflated the active-message stat - and the messages themselves stayed ack-pending on the server until AckWait expired. RabbitMQ requeues a closed channel's unacked deliveries immediately, so redelivery latency regressed from ~0 to up to AckWait on every consumer restart. Subscribe teardown now waits (bounded) for the consume loop to finish unsubscribing, flushes the connection so the server has processed the unsubscribe - a nak'd message could otherwise be redelivered straight into the dying subscription's still-registered pull request and strand until AckWait a second time - and then releases every in-flight message belonging to this consumer: entries are dropped from the map, and durable consumers' messages are nak'd for prompt redelivery. Ephemeral consumers skip the nak since the consumer is deleted on the same path. A delivery that loses the race against subscription shutdown is nak'd in the delivery handler instead of only being dropped from the map. This also deflakes TestDurableOffsetPinnedAtCreation: with prefetch 1, acking m0 let the server deliver m1 to the first subscription right before it was cancelled, leaving m1 ack-pending; the re-attach then timed out waiting for a redelivery that only AckWait would trigger (observed roughly 1 in 4 runs, now clean over repeated -count=25). Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 7 +++ internal/provider/connectors/natsjs/natsjs.go | 49 +++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 47 ++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 831d24b..fc93c96 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -100,6 +100,13 @@ The connector chooses the consumer kind from the source exits and churning transient clients cannot accumulate dead consumers against the server's per-stream consumer limit. +When a subscription ends, its delivered-but-unresolved messages are released: +their acks could only have arrived on the consume stream that just closed, so +the connector drops its claim on them and (for durable consumers) naks them +so they redeliver promptly. This matches RabbitMQ, which requeues a closed +channel's unacked deliveries immediately; without the nak they would only +redeliver after the full ack wait. + The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 connector's offset vocabulary so both accept the same values: `first`/`continue` -> deliver all, `last` -> the final message, `next` (or unset) -> deliver new, diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 4f2e905..6247653 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -598,7 +598,18 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c bd.consumeContexts.Add(source.GetName(), cc) defer func() { cc.Stop() + // Stop is asynchronous: wait (bounded) for the consume loop to finish + // unsubscribing, then flush so the server has processed the + // unsubscribe before the naks below — otherwise a nak'd message can + // be redelivered straight into this subscription's still-registered + // pull request and sit unclaimed until AckWait expires it again. + select { + case <-cc.Closed(): + case <-time.After(ephemeralDeleteTimeout): + } + _ = bd.nc.FlushTimeout(ephemeralDeleteTimeout) bd.consumeContexts.Delete(source.GetName()) + releaseInFlight(bd, streamName, cons.CachedInfo().Name, consCfg.Durable != "") if consCfg.Durable == "" { deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) } @@ -608,6 +619,39 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c return nil } +// releaseInFlight drops a just-ended subscription's unresolved deliveries from +// activeMessages. Acks travel on the consume stream that closed, so nothing +// can resolve these uuids anymore; left alone they would sit in the map for +// the life of the connection (a leak that also inflates the active-message +// stat). Durable consumers additionally get a best-effort Nak so the messages +// redeliver promptly — RabbitMQ requeues a closed channel's unacked deliveries +// immediately, whereas an untouched JetStream delivery waits out the full +// AckWait. Ephemeral consumers skip the Nak: the consumer is deleted on this +// same teardown path, which discards its delivery state anyway. A message the +// client resolves concurrently with teardown is deleted by whichever side gets +// there first; the loser's ack/nak fails and is ignored (at-least-once either +// way). +func releaseInFlight(bd *natsBrokerDetails, streamName, consumerName string, nak bool) { + for _, uuid := range bd.activeMessages.GetList() { + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + continue + } + m := mu.(jetstream.Msg) + md, err := m.Metadata() + if err != nil || md.Stream != streamName || md.Consumer != consumerName { + continue + } + bd.activeMessages.Delete(uuid) + if !nak { + continue + } + if err := m.Nak(); err != nil { + util.Logger.Debugf("natsjs: nak of in-flight message %s at subscription end: %s", uuid, err.Error()) + } + } +} + // deleteEphemeralConsumer removes an ephemeral consumer as soon as its // subscription ends. Ephemeral consumers do expire on their own after // InactiveThreshold, but until then each one still counts against the server's @@ -737,7 +781,12 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai case out <- pbmsg: atomic.AddInt64(&bd.consumed, 1) case <-ctx.Done(): + // The subscription ended before the server took the message: drop the + // claim and nak (best-effort) so a durable redelivers it promptly + // instead of after AckWait. Harmless for ephemerals, whose consumer is + // deleted on teardown. bd.activeMessages.Delete(uuid) + _ = m.Nak() } } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index be134d3..a08bd8f 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1020,6 +1020,53 @@ func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer was not deleted on teardown") } +// TestTeardownReleasesInFlightMessages: when a subscription ends, its +// delivered-but-unresolved messages are removed from the active-message map +// (their acks can only arrive on the consume stream that just closed, so they +// could never be resolved again — a leak for the life of the connection) and +// nak'd, so a durable redelivers them promptly instead of after the full +// AckWait, matching RabbitMQ's immediate requeue of a closed channel's +// unacked deliveries. +func TestTeardownReleasesInFlightMessages(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.inflight", Subjects: []string{"e"}} + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + // First subscription receives the backlog but never resolves it. + src := queueSource("events.inflight.consumer", "events.inflight", "e") + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, src, out) }() + for i := 0; i < n; i++ { + recv(t, out) // deliberately left unacked + } + cancel() + require.Nil(t, <-errCh) + + // The abandoned deliveries are no longer tracked... + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Zero(t, stats.Clients[0].ActiveMessages, + "teardown must release the subscription's in-flight messages") + + // ...and a re-subscribe gets them back well before AckWait (30s) because + // teardown nak'd them (recv's 10s timeout is the proof of promptness). + out2 := make(chan *pb.Message, n) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, src, out2) + for i := 0; i < n; i++ { + m := recv(t, out2) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + // TestDeclareOnlyEphemeralConsumerCleanedUp: DeclareOnly on a transient source // validates topology, but the ephemeral consumer it creates can never be // attached to afterwards (its server-generated name is not surfaced), so it is From 2b6d41bba6a32f2a83390410a3aa708bfbd002c7 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 14:20:58 -0400 Subject: [PATCH 20/77] feat: make the redelivery ack wait configurable The 30s AckWait was hardcoded. It sets one dial between two failure modes - a crashed client's in-flight messages are stuck for the full ack wait before another consumer gets them, while a healthy consumer that holds a message longer than the ack wait (including time queued in the client-side pull buffer behind a large prefetch) gets a duplicate redelivery - and the right value is deployment-specific: RabbitMQ's equivalent consumer timeout defaults to 30 minutes, so consumers doing tens of seconds of work per message are in-contract on RabbitMQ and silently double-processed here. Keep the failover-friendly 30s default and expose NATSJS_ACK_WAIT (Go duration), documented alongside the other stream/consumer knobs. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 11 ++++++++ internal/provider/connectors/natsjs/natsjs.go | 25 ++++++++++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 9 +++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index fc93c96..c03eba0 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -135,6 +135,16 @@ connector maps those onto JetStream primitives: | nack, `requeue_delay == 0`, DLA set | `DeadLetter` | publish to DLQ + `Term()` | | nack, `requeue_delay == 0`, no DLA | `Nack` | `msg.Nak()` | +A delivered message that is neither acked nor nacked redelivers after the +consumer's ack wait (`NATSJS_ACK_WAIT`, default 30s). That value sets one +dial between two failure modes: a crashed client's in-flight messages are +stuck until the ack wait passes before another consumer gets them (shorter +is better), while a healthy consumer that holds a message longer than the +ack wait — including time spent queued in the client-side pull buffer behind +a large prefetch — gets a duplicate redelivery (longer is better). The +default favors failover; RabbitMQ's equivalent consumer timeout defaults to +30 minutes, so deployments with legitimately slow consumers should raise it. + `NakWithDelay` is the native replacement for RabbitMQ's per-message-TTL + dead-letter retry-queue idiom; JetStream increments the delivery count, which the connector surfaces back to the client as the `x-retry-count` header (see @@ -221,6 +231,7 @@ forming a raft group per stream and per durable consumer: | `NATSJS_STREAM_MAX_AGE` | `72h` | Max age before messages are evicted (Go duration; `0` = keep forever). | | `NATSJS_STREAM_MAX_BYTES` | `0` (unlimited) | Hard per-stream storage cap in bytes. | | `NATSJS_API_TIMEOUT` | `30s` | Deadline for JetStream management API calls (stream / consumer creation). Go duration. | +| `NATSJS_ACK_WAIT` | `30s` | How long the server waits for an ack before redelivering a message. Go duration. | TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, `Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 6247653..a91dc66 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -41,6 +41,9 @@ const providerName = "natsjs" const retryCountHeaderName = "x-retry-count" const ( + // defaultAckWait is how long the server waits for an ack before + // redelivering a message. See ackWait for the trade-off it sets; + // override via NATSJS_ACK_WAIT. defaultAckWait = 30 * time.Second defaultInactiveThreshold = 5 * time.Minute defaultDedupWindow = 2 * time.Minute @@ -235,6 +238,26 @@ func jsAPITimeout() time.Duration { return defaultJSAPITimeout } +// ackWait is how long the server waits for a delivered message's ack before +// redelivering it. It sets one dial between two failure modes: a crashed +// client's in-flight messages are stuck for the full ack wait before another +// consumer gets them (shorter is better), while a healthy consumer that takes +// longer than the ack wait to process a message gets a duplicate redelivery +// (longer is better). The 30s default favors failover; deployments whose +// consumers legitimately hold messages longer — RabbitMQ's equivalent +// consumer timeout defaults to 30 minutes — should raise it via +// NATSJS_ACK_WAIT (Go duration). Note the pull buffer counts too: with a +// large prefetch a backlogged consumer can hold messages client-side longer +// than the ack wait just waiting their turn. +func ackWait() time.Duration { + if v := os.Getenv("NATSJS_ACK_WAIT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultAckWait +} + func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { clientID, err := GetClientIdentifier(ctx) if err != nil { @@ -520,7 +543,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: deliverPolicy, OptStartSeq: startSeq, // 0 (ignored) unless DeliverByStartSequence - AckWait: defaultAckWait, + AckWait: ackWait(), MaxAckPending: prefetch, } // Durable work-queue consumer for non-transient QUEUE / stream-group sources: diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index a08bd8f..a9cf61b 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -440,6 +440,15 @@ func TestJSAPITimeoutFromEnv(t *testing.T) { assert.Equal(t, defaultJSAPITimeout, jsAPITimeout()) } +func TestAckWaitFromEnv(t *testing.T) { + t.Setenv("NATSJS_ACK_WAIT", "5m") + assert.Equal(t, 5*time.Minute, ackWait()) + t.Setenv("NATSJS_ACK_WAIT", "bad") + assert.Equal(t, defaultAckWait, ackWait()) + t.Setenv("NATSJS_ACK_WAIT", "0") + assert.Equal(t, defaultAckWait, ackWait()) +} + func TestStreamRegistryCollapsesConcurrentCalls(t *testing.T) { r := newStreamRegistry() release := make(chan struct{}) From ef467c85635abc6dbd72370822044651f3ef0813 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 14:25:59 -0400 Subject: [PATCH 21/77] feat: enforce single-active-consumer with a pinned priority group SingleActiveConsumer sources only got a durable consumer, which is the opposite of single-active: every subscriber of the durable pulls from it, so instances compete and per-key ordering - the reason a client asks for SAC - silently breaks the moment a service scales past one replica. The parity table nevertheless claimed Native support. Map SAC onto JetStream's pinned-client priority group (nats-server 2.11+): the consumer is created with PriorityPolicyPinned and a single group, the server delivers only to the pinned client, standbys' pulls wait (still receiving idle heartbeats, so the liveness warning does not trip), and when the pinned client stops pulling for the configured TTL the pin moves to a standby. Failover time is bounded by NATSJS_SAC_PINNED_TTL (default 1m; it must comfortably exceed the ~30s pull re-issue cadence or the pin flaps between healthy standbys). The pull request must carry the group exactly when the consumer has one, so Consume keys off the effective (server-reported) config rather than the requested one: a server that predates priority groups drops the fields silently, and an existing durable the subscribe merely attached to may lack them - in both cases consumers compete as before, and a warning now says so instead of nothing. Priority config is update-mutable, so durables created before their source set SingleActiveConsumer upgrade in place, keeping name and ack position (covered by TestSingleActiveConsumerUpgradesExistingDurable; live pin-and-failover covered by TestSingleActiveConsumerPinsOneClient). Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 19 ++- internal/provider/connectors/natsjs/natsjs.go | 65 +++++++++- .../provider/connectors/natsjs/natsjs_test.go | 116 ++++++++++++++++++ 3 files changed, 194 insertions(+), 6 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index c03eba0..c18d83c 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -100,6 +100,22 @@ The connector chooses the consumer kind from the source exits and churning transient clients cannot accumulate dead consumers against the server's per-stream consumer limit. +A source with `SingleActiveConsumer` additionally maps onto a pinned-client +priority group (nats-server 2.11+): the server pins the first subscriber to +pull and delivers only to it, standbys' pulls wait, and when the pinned +client stops pulling for `NATSJS_SAC_PINNED_TTL` the pin moves to a standby. +That reproduces RabbitMQ's single-active-consumer semantics — ordered +processing across competing instances with automatic failover — natively, +with two caveats. Failover is bounded by the pinned TTL rather than +instantaneous on disconnect, and the TTL must comfortably exceed the pull +re-issue cadence (~30s with the client defaults) or the pin flaps. And on a +server that predates priority groups the config fields are silently dropped, +so single-active cannot be enforced; the connector detects that from the +effective consumer config and logs a warning that consumers will compete. +Priority-group config is update-mutable, so a durable created before its +source set `SingleActiveConsumer` is upgraded in place, keeping its name and +ack position. + When a subscription ends, its delivered-but-unresolved messages are released: their acks could only have arrived on the consume stream that just closed, so the connector drops its claim on them and (for durable consumers) naks them @@ -232,6 +248,7 @@ forming a raft group per stream and per durable consumer: | `NATSJS_STREAM_MAX_BYTES` | `0` (unlimited) | Hard per-stream storage cap in bytes. | | `NATSJS_API_TIMEOUT` | `30s` | Deadline for JetStream management API calls (stream / consumer creation). Go duration. | | `NATSJS_ACK_WAIT` | `30s` | How long the server waits for an ack before redelivering a message. Go duration. | +| `NATSJS_SAC_PINNED_TTL` | `1m` | Single-active-consumer failover deadline: how long the pinned client may go without pulling before a standby takes over. Go duration. | TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, `Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP @@ -250,7 +267,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Publish confirms | Native | `js.PublishMsg` returns a `PubAck`. | | Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | | Streams: offsets, start position | Native | JetStream is a log; `DeliverPolicy` maps `Offset`. | -| Single active consumer | Native | Consumer config (needs a durable name). | +| Single active consumer | Native | Pinned-client priority group on the durable (nats-server 2.11+); standby takes over within `NATSJS_SAC_PINNED_TTL`. | | Prefetch / QoS | Native | `MaxAckPending`; prefetch 0 (AMQP "unlimited") maps to unlimited (-1). | | HA / quorum queues | Native | JetStream R3 (Raft) via `NATSJS_STREAM_REPLICAS`. | | Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index a91dc66..60fa855 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -69,6 +69,14 @@ const ( // realistic consumer-outage backlog, short enough to stop indefinite // accumulation. Override via NATSJS_STREAM_MAX_AGE. defaultStreamMaxAge = 72 * time.Hour + // sacPriorityGroup is the priority group name used for single-active- + // consumer sources. One group is enough: RabbitMQ's SAC has no notion of + // multiple groups, and every subscriber of the durable joins this one. + sacPriorityGroup = "arke" + // defaultSACPinnedTTL is how long the server waits for a new pull request + // from the pinned (active) client before unpinning and failing over to a + // standby. See sacPinnedTTL; override via NATSJS_SAC_PINNED_TTL. + defaultSACPinnedTTL = time.Minute ) // supportedSourceOptionsList intentionally matches the amqp091 connector so that @@ -258,6 +266,22 @@ func ackWait() time.Duration { return defaultAckWait } +// sacPinnedTTL is the failover deadline for single-active-consumer sources: +// when the pinned (active) client sends no new pull request for this long, +// the server unpins it and the next standby's pull takes over. It must +// comfortably exceed the pull re-issue cadence (~30s with the client +// library's defaults) or the pin flaps between healthy standbys; lower it — +// together with faster pulls — only to shrink failover time. Configured via +// NATSJS_SAC_PINNED_TTL (Go duration). +func sacPinnedTTL() time.Duration { + if v := os.Getenv("NATSJS_SAC_PINNED_TTL"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultSACPinnedTTL +} + func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { clientID, err := GetClientIdentifier(ctx) if err != nil { @@ -554,6 +578,19 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // sources stay ephemeral and auto-expire after InactiveThreshold. if durable := durableName(source); durable != "" { consCfg.Durable = durable + if source.GetSingleActiveConsumer() { + // RabbitMQ's single-active-consumer delivers to one consumer at a + // time — the point is ordered processing across competing + // instances — and fails over when that consumer goes away. The + // JetStream equivalent is a pinned-client priority group + // (nats-server 2.11+): the server pins the first client to pull + // and standbys' pulls wait; when the pinned client stops pulling + // for PinnedTTL (or its pin is otherwise released) the next + // standby takes over. + consCfg.PriorityPolicy = jetstream.PriorityPolicyPinned + consCfg.PriorityGroups = []string{sacPriorityGroup} + consCfg.PinnedTTL = sacPinnedTTL() + } } else { consCfg.InactiveThreshold = defaultInactiveThreshold } @@ -598,20 +635,38 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c return nil } - cc, cerr := cons.Consume(func(m jetstream.Msg) { - p.handleDelivery(ctx, bd, source, m, out) - }, + consumeOpts := []jetstream.PullConsumeOpt{ // A short idle heartbeat plus an error handler keep a stalled delivery // path from failing silently: if the server stops serving this // consumer's pulls (broker restart, consumer raft leader not yet // serving after creation), the missed heartbeat is logged at warn and // the library re-issues its pull request after ~2x the heartbeat, - // instead of ~30s — and invisibly — with the defaults. + // instead of ~30s — and invisibly — with the defaults. Standby pulls + // of a pinned consumer receive idle heartbeats too, so single-active + // standbys do not trip this. jetstream.PullHeartbeat(defaultConsumeHeartbeat), jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { util.Logger.Warn("natsjs: consume error on source {0}: {1}", source.GetName(), err.Error()) }), - ) + } + // Pull requests must carry the priority group exactly when the consumer + // has one, so key off the effective (server-reported) config, not the + // requested one: a server too old for priority groups (< 2.11) silently + // drops the fields, and an existing durable the subscribe merely attached + // to may predate them. In either case single-active cannot be enforced — + // consumers compete, today's behavior — and that degradation must be + // visible, not silent. + if groups := cons.CachedInfo().Config.PriorityGroups; len(groups) > 0 { + consumeOpts = append(consumeOpts, jetstream.PullPriorityGroup(groups[0])) + } else if consCfg.PriorityPolicy == jetstream.PriorityPolicyPinned { + util.Logger.Warn( + "natsjs: source {0} requested a single active consumer but the consumer has no priority group "+ + "(broker predates them, or an existing durable could not be updated); consumers will compete", + source.GetName()) + } + cc, cerr := cons.Consume(func(m jetstream.Msg) { + p.handleDelivery(ctx, bd, source, m, out) + }, consumeOpts...) if cerr != nil { if consCfg.Durable == "" { deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index a9cf61b..32ee45e 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -449,6 +449,15 @@ func TestAckWaitFromEnv(t *testing.T) { assert.Equal(t, defaultAckWait, ackWait()) } +func TestSACPinnedTTLFromEnv(t *testing.T) { + t.Setenv("NATSJS_SAC_PINNED_TTL", "10s") + assert.Equal(t, 10*time.Second, sacPinnedTTL()) + t.Setenv("NATSJS_SAC_PINNED_TTL", "bad") + assert.Equal(t, defaultSACPinnedTTL, sacPinnedTTL()) + t.Setenv("NATSJS_SAC_PINNED_TTL", "0") + assert.Equal(t, defaultSACPinnedTTL, sacPinnedTTL()) +} + func TestStreamRegistryCollapsesConcurrentCalls(t *testing.T) { r := newStreamRegistry() release := make(chan struct{}) @@ -1076,6 +1085,113 @@ func TestTeardownReleasesInFlightMessages(t *testing.T) { } } +// sacSource is queueSource with SingleActiveConsumer set. +func sacSource(name, addr string, subjects ...string) *pb.Source { + s := queueSource(name, addr, subjects...) + s.SingleActiveConsumer = true + return s +} + +// TestSingleActiveConsumerPinsOneClient: a SingleActiveConsumer source maps to +// a pinned-client priority group, so with two live subscribers exactly one +// receives (RabbitMQ SAC semantics — ordered processing across competing +// instances) and the standby takes over after the active one goes away and +// its pin expires. +func TestSingleActiveConsumerPinsOneClient(t *testing.T) { + t.Setenv("NATSJS_SAC_PINNED_TTL", "1s") // fast failover for the test + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sac", Subjects: []string{"e"}} + src := sacSource("events.sac.consumer", "events.sac", "e") + + // The first subscriber pulls alone, so it takes the pin. + outA := make(chan *pb.Message, 8) + ctxA, cancelA := context.WithCancel(ctx) + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src, outA) }() + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, outA) + assert.Equal(t, "m0", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // A standby joins. Everything published while the pin holder is alive + // must keep going to the pin holder only. + outB := make(chan *pb.Message, 8) + ctxB, cancelB := context.WithCancel(ctx) + defer cancelB() + go p.Subscribe(ctxB, src, outB) + time.Sleep(500 * time.Millisecond) // let the standby's pull reach the server + + for i := 1; i <= 3; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + for i := 1; i <= 3; i++ { + m := recv(t, outA) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + select { + case got := <-outB: + t.Fatalf("standby received %q while another consumer held the pin", got.GetBody()) + case <-time.After(500 * time.Millisecond): + } + + // The active subscriber goes away; once its pin expires (1s here) the + // standby is pinned and receives. + cancelA() + require.Nil(t, <-errA) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m4")})) + m = recv(t, outB) + assert.Equal(t, "m4", string(m.GetBody()), "standby takes over after the active consumer goes away") + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} + +// TestSingleActiveConsumerUpgradesExistingDurable: priority-group config is +// update-mutable, so a durable created before SingleActiveConsumer was set on +// the source is upgraded in place — same name, same ack position — instead of +// failing the subscribe. +func TestSingleActiveConsumerUpgradesExistingDurable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacup", Subjects: []string{"e"}} + plain := queueSource("events.sacup.consumer", "events.sacup", "e") + + // Create the durable without single-active (an older deployment). + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, plain, out) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe with SingleActiveConsumer set: the consumer is updated in + // place and keeps delivering from its stored position. + sac := sacSource("events.sacup.consumer", "events.sacup", "e") + out2 := make(chan *pb.Message, 1) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, sac, out2) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + m = recv(t, out2) + assert.Equal(t, "m1", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // The effective config now carries the pinned priority group. + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.sacup")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(sac)) + require.NoError(t, cerr) + assert.Equal(t, []string{sacPriorityGroup}, cons.CachedInfo().Config.PriorityGroups) + assert.Equal(t, jetstream.PriorityPolicyPinned, cons.CachedInfo().Config.PriorityPolicy) +} + // TestDeclareOnlyEphemeralConsumerCleanedUp: DeclareOnly on a transient source // validates topology, but the ephemeral consumer it creates can never be // attached to afterwards (its server-generated name is not surfaced), so it is From f8c1a97c28a666f767bd3394e7611212fe6f43b1 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sat, 11 Jul 2026 14:26:50 -0400 Subject: [PATCH 22/77] perf: reuse the consumer info SourceStats already fetched stream.Consumer() fetches the consumer's info to build the handle, and SourceStats then called Info() on it - a second, identical round-trip to the JetStream API on every stats poll of every durable source. CachedInfo() returns the info the handle was just built from. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 60fa855..2c8c3d9 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1067,14 +1067,15 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb // consumer (or before it exists), fall back to the stream view. if durable := durableName(source); durable != "" { if cons, cerr := stream.Consumer(ctx, durable); cerr == nil { - if ci, ierr := cons.Info(ctx); ierr == nil { - stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec - stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec - // Delivered.Consumer counts deliveries (redeliveries included, - // like RabbitMQ's deliver rate). Ephemeral sources have no - // consumer identity to sample here, so their rate stays zero. - stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable, now, ci.Delivered.Consumer) - } + // Consumer() fetched fresh info to return the handle; reuse it + // instead of paying a second round-trip on every stats poll. + ci := cons.CachedInfo() + stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec + stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec + // Delivered.Consumer counts deliveries (redeliveries included, + // like RabbitMQ's deliver rate). Ephemeral sources have no + // consumer identity to sample here, so their rate stays zero. + stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable, now, ci.Delivered.Consumer) } } return stats From 32622669cc9963fa484ae2cbc53c5bdec0006a82 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:10:53 -0400 Subject: [PATCH 23/77] fix: name single-active durables after their ConsumerGroup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single-active source's instances coordinate by attaching to one shared durable, so the durable must be named for the coordination identity, not the subscriber. That identity is the ConsumerGroup option: sources that share one name but split their subscribers into groups — one independently ordered group per partition of a shared stream — previously collapsed onto a single durable named after the source, so one group's client took the pin and every other group starved. amqp091 already uses ConsumerGroup as the single-active consumer reference for stream sources, and rejects a single-active stream source that names no group; natsjs now matches both. A single-active queue source still falls back to its own name — all instances of a single-active queue share the queue by definition. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 12 +++- .../provider/connectors/natsjs/helpers.go | 20 ++++++ .../connectors/natsjs/helpers_test.go | 17 +++++ internal/provider/connectors/natsjs/natsjs.go | 11 ++++ .../provider/connectors/natsjs/natsjs_test.go | 63 +++++++++++++++++++ 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index c18d83c..11b1681 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -104,6 +104,16 @@ A source with `SingleActiveConsumer` additionally maps onto a pinned-client priority group (nats-server 2.11+): the server pins the first subscriber to pull and delivers only to it, standbys' pulls wait, and when the pinned client stops pulling for `NATSJS_SAC_PINNED_TTL` the pin moves to a standby. +The durable such a source attaches to is named after its `ConsumerGroup` +option when set: the group is the identity the instances coordinate through +(amqp091 uses it as the single-active consumer reference the same way), so +sources that share one name but split their subscribers into groups — one +independently ordered group per partition of a shared stream — get one +pinned consumer per group instead of collapsing onto a single pin that +starves every other group. Without a group, a single-active queue source +falls back to its own name (all instances of a single-active queue share the +queue by definition), and a single-active stream source is rejected at +subscribe time, as in amqp091. That reproduces RabbitMQ's single-active-consumer semantics — ordered processing across competing instances with automatic failover — natively, with two caveats. Failover is bounded by the pinned TTL rather than @@ -292,7 +302,7 @@ amqp091 connector, so existing client sources validate unchanged: | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing key for dead-lettered messages. | | `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | -| `ConsumerGroup` | string | Durable consumer group name (Streams only). | +| `ConsumerGroup` | string | Durable consumer group name (stream sources; also names the durable that single-active instances coordinate through, and is required for a single-active stream source). | ## Known limitations diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index cb761c1..20480bd 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -219,6 +219,26 @@ func durableName(source *pb.Source) string { return "" } if source.GetSingleActiveConsumer() { + // Single-active instances coordinate by attaching to one shared + // consumer, so the durable must be named for the coordination + // identity, not the individual subscriber. That identity is the + // ConsumerGroup option when set — amqp091 uses it as the consumer + // reference for single-active stream sources for the same reason — + // which also lets sources that share one name but separate their + // instances only by group (one group per partition/tenant/shard of a + // shared stream) get one durable per group instead of collapsing + // onto a single pinned consumer that starves every other group. + if cg := source.GetOptions()["ConsumerGroup"]; cg != "" { + return streamNameFor(cg) + } + // A stream source with no group has no coordination identity — + // Subscribe rejects it, like amqp091. A queue source falls back to + // the queue's own name: all instances of a single-active queue share + // it by definition (RabbitMQ's x-single-active-consumer is a queue + // property). + if source.GetType() == pb.Source_STREAM { + return "" + } return streamNameFor(source.GetName()) } switch source.GetType() { diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index a00c84b..9eea2df 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -160,6 +160,23 @@ func TestDurableName(t *testing.T) { sac.SingleActiveConsumer = true assert.Equal(t, "arke_sac", durableName(sac)) + // SingleActiveConsumer with a ConsumerGroup -> the group is the + // coordination identity, so it names the durable (amqp091 uses the group + // as the single-active consumer reference the same way). Sources sharing + // a name but split into groups must NOT collapse onto one durable. + sacg := q("sac") + sacg.SingleActiveConsumer = true + sacg.Options = map[string]string{"ConsumerGroup": "grp.b"} + assert.Equal(t, "arke_grp_b", durableName(sacg)) + + // single-active STREAM source without a ConsumerGroup: no coordination + // identity exists; Subscribe rejects it (amqp091 parity), and there is no + // durable name for it + sacs := q("sacs") + sacs.SingleActiveConsumer = true + sacs.Type = pb.Source_STREAM + assert.Equal(t, "", durableName(sacs)) + // STREAM with ConsumerGroup -> durable (named after the group) st := q("st") st.Type = pb.Source_STREAM diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 2c8c3d9..eb1191c 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -534,6 +534,17 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c "natsjs: source {0} sets {1}, which natsjs accepts but does not apply; retention is stream-wide (NATSJS_STREAM_MAX_AGE / NATSJS_STREAM_MAX_BYTES)", source.GetName(), strings.Join(unapplied, ", ")) } + // A single-active stream source must name the ConsumerGroup its instances + // coordinate through (see durableName); amqp091 rejects this combination + // with the same message. Without a group each subscriber would get its own + // durable and they would all compete — the opposite of single-active. + if source.GetSingleActiveConsumer() && source.GetType() == pb.Source_STREAM && + source.GetOptions()["ConsumerGroup"] == "" { + return &pb.Error{ + Message: fmt.Sprintf("source %q: single active consumer requested but no ConsumerGroup option set", source.GetName()), + IsFatal: true, + } + } addr := source.GetAddress().GetName() streamName, serr := p.ensureStream(ctx, bd, addr) if serr != nil { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 32ee45e..02a65e2 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1192,6 +1192,69 @@ func TestSingleActiveConsumerUpgradesExistingDurable(t *testing.T) { assert.Equal(t, jetstream.PriorityPolicyPinned, cons.CachedInfo().Config.PriorityPolicy) } +// TestSingleActiveConsumerGroupsAreIndependent: the ConsumerGroup option is +// the identity single-active instances coordinate through (amqp091 uses it as +// the consumer reference the same way), so two groups of a source — same +// source name, different ConsumerGroup — must get independent pinned durables +// that each receive the whole stream. Naming the durable after the source +// name instead would collapse every group onto one pinned consumer and starve +// all groups but the pin holder's. +func TestSingleActiveConsumerGroupsAreIndependent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacgroups", Subjects: []string{"e"}} + mkSrc := func(group string) *pb.Source { + return &pb.Source{ + Name: "events.sacgroups.consumer", // shared across groups + Type: pb.Source_STREAM, + SingleActiveConsumer: true, + Address: &pb.Address{Name: "events.sacgroups", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first", "ConsumerGroup": "grp-" + group}, + } + } + + const n = 2 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + outA := make(chan *pb.Message, n) + outB := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, mkSrc("a"), outA) + go p.Subscribe(subCtx, mkSrc("b"), outB) + + // Each group replays the full backlog independently. + for i := 0; i < n; i++ { + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + } + for i := 0; i < n; i++ { + require.Nil(t, p.Ack(ctx, recv(t, outB).GetUuid())) + } +} + +// TestSingleActiveStreamSourceRequiresConsumerGroup: a single-active stream +// source names no ConsumerGroup to coordinate through — amqp091 rejects the +// combination, and so does natsjs, instead of inventing a per-subscriber +// durable that would make the instances compete. +func TestSingleActiveStreamSourceRequiresConsumerGroup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.sacnogroup.consumer", + Type: pb.Source_STREAM, + SingleActiveConsumer: true, + Address: &pb.Address{Name: "events.sacnogroup", Subjects: []string{"e"}}, + } + perr := p.Subscribe(ctx, src, make(chan *pb.Message, 1)) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "no ConsumerGroup option set") + assert.True(t, perr.GetIsFatal()) +} + // TestDeclareOnlyEphemeralConsumerCleanedUp: DeclareOnly on a transient source // validates topology, but the ephemeral consumer it creates can never be // attached to afterwards (its server-generated name is not surfaced), so it is From cee73e951c286cba0993bcc4c4980bbb91757658 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:14:08 -0400 Subject: [PATCH 24/77] fix: collapse redundant bindings JetStream rejects as overlapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An AMQP binding set may be redundant — a wildcard binding alongside a specific key it already covers, like orders.# with orders.created — and that is harmless on a broker that routes a message to a queue once no matter how many bindings match. JetStream instead rejects a consumer whose filter subjects overlap (one being a subset of another), so any source with such a binding set failed its subscribe outright. Collapse the mapped filters to the widest set before creating the consumer: the surviving wildcard filter matches everything the dropped filters did, and intersecting patterns where neither contains the other still pass through untouched (JetStream accepts those). Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 ++++ .../provider/connectors/natsjs/helpers.go | 47 ++++++++++++++++++- .../connectors/natsjs/helpers_test.go | 46 ++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 45 ++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 11b1681..6dce2b4 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -68,6 +68,15 @@ subjects — so wildcard characters are sanitized to `_` like any other illegal character. In address names, empty tokens and tokens equal to `~` are replaced with `_` rather than dropped, so distinct names keep distinct roots. +A redundant binding set — a wildcard binding alongside a specific key it +already covers, such as `orders.#` with `orders.created` — is legal in AMQP, +where a message is routed to the queue once no matter how many bindings +match. JetStream instead rejects a consumer whose filter subjects overlap +(one being a subset of another), so the connector collapses the mapped +filters to the widest set before creating the consumer; the surviving +wildcard filter matches everything the dropped filters did. Intersecting +patterns where neither contains the other are kept as-is. + This subject scheme is part of the connector's persistence contract: retained messages are stored under these subjects for the life of the stream's retention limits, so any future change to the encoding is a diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 20480bd..1d4496c 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -198,7 +198,52 @@ func filterSubjectsFor(source *pb.Source) []string { add(prefix + "." + pat) } } - return out + // An AMQP binding set may be redundant — "orders.#" alongside + // "orders.created" — which is harmless on a broker that routes a message + // to a queue once no matter how many of its bindings match. JetStream + // instead rejects a consumer whose filter subjects overlap (one being a + // subset of another), so drop every filter another filter already covers. + // The surviving wider filter matches everything the dropped one did. + kept := make([]string, 0, len(out)) + for _, s := range out { + subsumed := false + for _, t := range out { + if t != s && subjectSubsumes(t, s) { + subsumed = true + break + } + } + if !subsumed { + kept = append(kept, s) + } + } + return kept +} + +// subjectSubsumes reports whether every subject matched by narrow is also +// matched by wide (both may contain NATS wildcards): a wide '>' covers any +// one or more remaining tokens, and a wide '*' covers a literal token or +// another '*' but never '>'. This is the subset relation JetStream applies +// when it rejects a consumer's filter subjects as overlapping. +func subjectSubsumes(wide, narrow string) bool { + w := strings.Split(wide, ".") + n := strings.Split(narrow, ".") + for i, wt := range w { + if i >= len(n) { + return false + } + if wt == ">" { + return true + } + nt := n[i] + if nt == ">" || (nt == "*" && wt != "*") { + return false + } + if wt != "*" && wt != nt { + return false + } + } + return len(w) == len(n) } // durableName returns the JetStream durable consumer name for a source, or "" diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 9eea2df..964c4a9 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -129,6 +129,52 @@ func TestFilterSubjectsFor(t *testing.T) { assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, filterSubjectsFor(src("#", ""))) + // redundant bindings collapse to the widest filters: JetStream rejects a + // consumer whose filter subjects overlap, while AMQP happily routes a + // message once however many bindings match + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#", "orders.created"))) + assert.Equal(t, + []string{"events.orders.~.region", "events.orders.~.region.>"}, + filterSubjectsFor(src("region.#", "region.us", "region.us.updated"))) + assert.Equal(t, + []string{"events.orders.~.*"}, + filterSubjectsFor(src("*", "created"))) + // intersecting patterns where neither contains the other both survive + // (JetStream accepts them) + assert.Equal(t, + []string{"events.orders.~.*.b", "events.orders.~.a.*"}, + filterSubjectsFor(src("*.b", "a.*"))) +} + +func TestSubjectSubsumes(t *testing.T) { + cover := func(wide, narrow string) bool { return subjectSubsumes(wide, narrow) } + + assert.True(t, cover("a.>", "a.b")) + assert.True(t, cover("a.>", "a.b.c")) + assert.True(t, cover("a.>", "a.b.>")) + assert.True(t, cover("a.>", "a.*")) + assert.True(t, cover("a.*", "a.b")) + assert.True(t, cover("a.*.c", "a.b.c")) + assert.True(t, cover(">", "anything.at.all")) + + // '>' needs at least one token, so it does not cover the bare base + assert.False(t, cover("a.>", "a")) + // a bounded pattern cannot cover an unbounded one + assert.False(t, cover("a.*", "a.>")) + assert.False(t, cover("a.b.>", "a.>")) + // '*' does not cover a different literal position count + assert.False(t, cover("a.*", "a.b.c")) + assert.False(t, cover("a.b", "a")) + assert.False(t, cover("a", "a.b")) + // literal mismatch + assert.False(t, cover("a.b", "a.c")) + // a literal never covers a wildcard + assert.False(t, cover("a.b", "a.*")) + // intersecting but neither is a subset + assert.False(t, cover("*.b", "a.*")) + assert.False(t, cover("a.*", "*.b")) } func TestDurableName(t *testing.T) { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 02a65e2..bd76675 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1085,6 +1085,51 @@ func TestTeardownReleasesInFlightMessages(t *testing.T) { } } +// TestRedundantBindingsCoexist: an AMQP binding set may be redundant — a +// wildcard binding alongside a specific key it already covers — and RabbitMQ +// routes each message to the queue once no matter how many bindings match. +// JetStream rejects a consumer whose filter subjects overlap, so the +// connector must collapse the redundant filters instead of failing the +// subscribe (and each message still arrives exactly once). +func TestRedundantBindingsCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + pub := func(key, body string) { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.redundant", Subjects: []string{key}}, + Body: []byte(body), + })) + } + pub("orders.created", "covered-by-both") + pub("other.thing", "covered-by-wildcard") + + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + errCh := make(chan *pb.Error, 1) + go func() { + errCh <- p.Subscribe(subCtx, + queueSource("events.redundant.consumer", "events.redundant", "#", "orders.created"), out) + }() + + got := map[string]int{} + for i := 0; i < 2; i++ { + m := recv(t, out) + got[string(m.GetBody())]++ + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.Equal(t, map[string]int{"covered-by-both": 1, "covered-by-wildcard": 1}, got) + + select { + case extra := <-out: + t.Fatalf("message delivered twice: %q", extra.GetBody()) + case perr := <-errCh: + t.Fatalf("subscribe failed on redundant bindings: %s", perr.GetMessage()) + case <-time.After(500 * time.Millisecond): + } +} + // sacSource is queueSource with SingleActiveConsumer set. func sacSource(name, addr string, subjects ...string) *pb.Source { s := queueSource(name, addr, subjects...) From 6b9f6ccb6d368fd038f5f3d02f6c033898024126 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:15:42 -0400 Subject: [PATCH 25/77] fix: carry the retry count on dead-letter copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RabbitMQ dead-letters broker-side and stamps the copy with its death trail (x-death), so a DLQ consumer can see how many delivery attempts preceded the move. The natsjs dead-letter is a proxy-side republish that carried only the original headers — the delivery history vanished. Stamp the DLQ copy with the same x-retry-count the consumer saw when it gave the message up, synthesized from the delivery count exactly like the delivery path does. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 5 +++- internal/provider/connectors/natsjs/natsjs.go | 30 ++++++++++++++----- .../provider/connectors/natsjs/natsjs_test.go | 11 ++++++- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 6dce2b4..322ec97 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -193,7 +193,10 @@ the message ack-pending — the server then falls back to a nack, so the message is redelivered and dead-lettering is retried instead of the message being lost. The DLQ copy carries a `Nats-Msg-Id` derived from the original's stream sequence, so a retried dead-letter of the same message deduplicates in -the DLQ within its dedup window. +the DLQ within its dedup window, and the `x-retry-count` the consumer saw +when it gave the message up — RabbitMQ's broker-side move preserves the +death trail in `x-death`, and the retry count is this connector's equivalent +(a plain republish would lose it). ## Retry-count header diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index eb1191c..2faca28 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -943,7 +943,9 @@ func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, d // redelivery while also absent from the DLQ. The DLQ publish carries a // Nats-Msg-Id derived from the original's stream sequence, so a retried // dead-letter of the same message cannot duplicate it in the DLQ within the -// dedup window. +// dedup window, and the x-retry-count the consumer saw when it gave the +// message up (the proxy-side analogue of the x-death trail RabbitMQ's +// broker-side move preserves). func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid string) *pb.Error { bd, err := p.getBrokerDetails(ctx) if err != nil { @@ -962,17 +964,29 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid uuid, dla, serr.Error()) return &pb.Error{Message: fmt.Sprintf("dead-letter ensure stream: %s", serr.Error())} } - dlqMsg := &nats.Msg{ - Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), - Data: m.Data(), - // Copied so the publish options below do not mutate the original's - // header map. - Header: copyHeader(m.Headers()), - } + // Copied so neither the retry count below nor the publish options + // mutate the original's header map. + dlqHeader := copyHeader(m.Headers()) var pubOpts []jetstream.PublishOpt if md, merr := m.Metadata(); merr == nil { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("dlq-%s-%d", md.Stream, md.Sequence.Stream))) + // RabbitMQ stamps a dead-lettered copy broker-side (x-death) with + // how it died; the JetStream copy is a plain publish that would + // otherwise lose that trail. Carry the same retry count the + // consumer saw when it gave the message up, synthesized exactly + // like handleDelivery does for deliveries. + if md.NumDelivered > 1 { + if dlqHeader == nil { + dlqHeader = nats.Header{} + } + dlqHeader.Set(retryCountHeaderName, strconv.FormatUint(md.NumDelivered-1, 10)) + } + } + dlqMsg := &nats.Msg{ + Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), + Data: m.Data(), + Header: dlqHeader, } if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { util.Logger.Warn( diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index bd76675..ffb9707 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -212,7 +212,12 @@ func TestDeadLetter(t *testing.T) { defer cancel() go p.Subscribe(subCtx, src, out) + // Fail the first delivery, then dead-letter the redelivery, so the copy + // has a retry trail to carry. + first := recv(t, out) + require.Nil(t, p.Nack(ctx, first.GetUuid())) m := recv(t, out) + require.Equal(t, "1", m.GetHeaders()[retryCountHeaderName], "redelivery precondition") require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) // The dead-lettered copy lands on the DLQ stream. @@ -224,13 +229,17 @@ func TestDeadLetter(t *testing.T) { assert.Equal(t, int64(1), dlqStats.GetMessageCount()) // The copy carries a deterministic Nats-Msg-Id, so a retried dead-letter of - // the same message dedups in the DLQ instead of duplicating. + // the same message dedups in the DLQ instead of duplicating — and the same + // retry count the consumer saw when it gave the message up (RabbitMQ's + // broker-side dead-lettering preserves the death trail via x-death). bd := p.getBrokerDetailsByIdentifier("test-client") dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlq")) require.NoError(t, serr) raw, gerr := dlqStream.GetMsg(ctx, 1) require.NoError(t, gerr) assert.NotEmpty(t, raw.Header.Get("Nats-Msg-Id"), "DLQ copy must carry a dedup message id") + assert.Equal(t, "1", raw.Header.Get(retryCountHeaderName), + "DLQ copy must carry the retry count at dead-letter time") } // TestDeadLetterFailureKeepsMessage: when the dead-letter publish cannot happen From 5b0bf2860529ad97c689e1803588e4413e40a1eb Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:17:05 -0400 Subject: [PATCH 26/77] fix: sample SourceStats rates per polling source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish/deliver rate baselines were keyed by stream (and durable) only, but several sources commonly share one address — many queues bound to one exchange — and each polls its stats on its own cadence. With a shared key, every poll resets the window for all of them, so any poll arriving shortly after another source's reports the rate over that tiny gap: effectively zero, or wildly noisy for bursty traffic. Key the samples by the polling source as well, so each source differences the shared counters over its own poll interval. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 11 +++++-- .../provider/connectors/natsjs/natsjs_test.go | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 2faca28..fa1471c 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1082,7 +1082,12 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb // JetStream exposes no rates, only counters; sample them between calls // (see rateTracker). The stream's LastSeq counts every publish to the // address root, so the publish rate is per address, not per binding. - stats.PublishRate = bd.rates.observe("pub/"+streamName, now, info.State.LastSeq) + // The sample key includes the polling source: the counters are shared, + // but each source polls on its own cadence, and a shared key would make + // every observation window the gap since whichever source polled last — + // with several sources on one address the later polls of a cycle would + // report rates over milliseconds instead of the poll interval. + stats.PublishRate = bd.rates.observe("pub/"+streamName+"/"+source.GetName(), now, info.State.LastSeq) // For a source with a durable consumer, report that consumer's actual // backlog (undelivered + delivered-but-unacked) instead of the stream // depth: the stream retains acked messages under its retention limits, so @@ -1100,7 +1105,9 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb // Delivered.Consumer counts deliveries (redeliveries included, // like RabbitMQ's deliver rate). Ephemeral sources have no // consumer identity to sample here, so their rate stays zero. - stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable, now, ci.Delivered.Consumer) + // Keyed per polling source like the publish rate above (sources + // can share a durable through a common ConsumerGroup). + stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable+"/"+source.GetName(), now, ci.Delivered.Consumer) } } return stats diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index ffb9707..47636de 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1389,6 +1389,39 @@ func TestSourceStatsRates(t *testing.T) { assert.Positive(t, stats.GetDeliverRate(), "5 deliveries since the last scrape") } +// TestSourceStatsRatesIndependentPerSource: several sources commonly share +// one address (many queues bound to one exchange), and each polls its stats +// on its own cadence. The rate baselines must therefore be per source: with +// a shared per-stream key, any other source's poll resets the window, and a +// poll right after it reports the rate over that tiny gap — here, zero — +// instead of over the poller's own interval. +func TestSourceStatsRatesIndependentPerSource(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.ratesplit", Subjects: []string{"e"}} + srcA := queueSource("events.ratesplit.a", "events.ratesplit", "e") + srcB := queueSource("events.ratesplit.b", "events.ratesplit", "e") + + // Create the stream and baseline source A. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.SourceStats(ctx, srcA).GetError()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + } + + // Source B polls (its own baseline) between A's polls. A's next poll must + // still see the 5 publishes inside its own window. + require.Nil(t, p.SourceStats(ctx, srcB).GetError()) + time.Sleep(20 * time.Millisecond) + + stats := p.SourceStats(ctx, srcA) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), + "another source's poll must not reset this source's rate window") +} + func TestUnappliedSourceOptions(t *testing.T) { src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } From 40fe392f532a0529f0d50ba73b3777884e5788c4 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:41:27 -0400 Subject: [PATCH 27/77] fix: recover when a stream is deleted out from under a connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connector memoizes streams it has ensured for the life of the connection, but a NATS client connection outlives broker state changes that would sever an AMQP connection: an operator can delete a stream (a reset, a storage wipe, external cleanup) while every publisher keeps its connection and keeps failing with 'no response from stream' — forever, because nothing ever invalidates the memoized entry. The AMQP world never sees this failure mode: wiping broker state means restarting the broker, which drops TCP connections and forces every client to reconnect and re-declare. Now the paths that trip over the missing stream heal it: a publish that gets ErrNoStreamResponse drops the memoized entry, re-ensures the stream, and retries once (the failed attempt was not stored — no stream answered — so the retry cannot duplicate); a subscribe that finds the stream missing does the same instead of returning a fatal error until the client reconnects; a dead-letter publish that hits the same error drops the entry so the retry that follows the returned error (the server nacks, the message redelivers) re-creates the DLQ stream. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 12 ++- internal/provider/connectors/natsjs/natsjs.go | 39 ++++++++- .../provider/connectors/natsjs/natsjs_test.go | 82 +++++++++++++++++++ 3 files changed, 129 insertions(+), 4 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 322ec97..2ac834a 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -239,7 +239,7 @@ is intended as the primary disk guard. ## Operational resilience -Three mechanisms harden the connector against a cold or busy broker. All are +Four mechanisms harden the connector against a cold or busy broker. All are sized for a clustered (replicated) server, where creating topology also means forming a raft group per stream and per durable consumer: @@ -260,6 +260,16 @@ forming a raft group per stream and per durable consumer: the missed heartbeat is logged at warn level and the client re-issues its pull request after roughly twice the heartbeat. With library defaults the same stall would go unlogged and take ~30s per detection cycle. +- **Stale-topology recovery.** A stream can be deleted out from under a + live connection — an operator reset, a storage wipe — and a NATS client + outlives broker state changes that would sever an AMQP connection, so the + memoized "already ensured" answer can go stale. When a publish gets "no + response from stream" or a subscribe finds the stream missing, the + connection drops its memoized entry, re-asserts the stream, and retries + once (the failed publish attempt was not stored, so the retry cannot + duplicate it), instead of failing every call until the client reconnects. + A failed dead-letter publish likewise drops the entry so the retry that + follows re-creates the DLQ stream. ## Configuration diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index fa1471c..43963cd 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -497,7 +497,8 @@ func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Er func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message) *pb.Error { addr := msg.GetAddress() - if _, err := p.ensureStream(ctx, bd, addr.GetName()); err != nil { + streamName, err := p.ensureStream(ctx, bd, addr.GetName()) + if err != nil { return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} } nmsg := &nats.Msg{ @@ -510,8 +511,23 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, if msg.GetPublisherName() != "" || msg.GetPublishId() != 0 { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) } - if _, err := bd.js.PublishMsg(ctx, nmsg, pubOpts...); err != nil { - return &pb.Error{Message: fmt.Sprintf("publish: %s", err.Error())} + _, perr := bd.js.PublishMsg(ctx, nmsg, pubOpts...) + if errors.Is(perr, jetstream.ErrNoStreamResponse) { + // No stream captured the subject even though this connection ensured + // one: the stream has been deleted underneath the connection (an + // operator reset, a storage wipe). The memoized entry would pin that + // stale answer for the connection's lifetime, failing every subsequent + // publish — and a NATS client outlives broker state changes that would + // sever an AMQP connection. Drop the entry, re-assert the stream, and + // retry once; the first attempt was not stored (no stream answered), + // so the retry cannot duplicate it. + bd.knownStreams.Delete(streamName) + if _, rerr := p.ensureStream(ctx, bd, addr.GetName()); rerr == nil { + _, perr = bd.js.PublishMsg(ctx, nmsg, pubOpts...) + } + } + if perr != nil { + return &pb.Error{Message: fmt.Sprintf("publish: %s", perr.Error())} } atomic.AddInt64(&bd.produced, 1) return nil @@ -557,6 +573,16 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c tctx, tcancel := context.WithTimeout(ctx, jsAPITimeout()) defer tcancel() stream, serr := bd.js.Stream(tctx, streamName) + if errors.Is(serr, jetstream.ErrStreamNotFound) { + // Same stale-memo case as the publish path: the stream was deleted out + // from under the connection after this connection ensured it. Drop the + // entry and re-assert the stream instead of failing the subscribe until + // the client reconnects. + bd.knownStreams.Delete(streamName) + if _, rerr := p.ensureStream(ctx, bd, addr); rerr == nil { + stream, serr = bd.js.Stream(tctx, streamName) + } + } if serr != nil { return &pb.Error{Message: fmt.Sprintf("open stream: %s", serr.Error()), IsFatal: true} } @@ -989,6 +1015,13 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid Header: dlqHeader, } if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { + if errors.Is(perr, jetstream.ErrNoStreamResponse) { + // The DLQ stream was deleted after this connection ensured it. + // Drop the memoized entry so the dead-letter retry that follows + // the returned error (the server nacks, the message redelivers) + // re-creates the stream instead of failing the same way forever. + bd.knownStreams.Delete(streamNameFor(dla)) + } util.Logger.Warn( "natsjs: dead-letter of message {0} failed (publish to {1}: {2}); message stays in flight", uuid, dlqMsg.Subject, perr.Error()) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 47636de..1501ead 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -288,6 +288,88 @@ func TestDeadLetterFailureKeepsMessage(t *testing.T) { // convention — the amqp091 connector honors it by leaving the channel // default, which is unlimited — so it maps to JetStream's unlimited (-1), // not to a one-message-at-a-time clamp. +// A stream can be deleted out from under a live connection — an operator +// reset, a storage wipe, external cleanup — without the NATS connection +// noticing, and knownStreams pins the "already ensured" answer for the +// connection's lifetime. Publish, Subscribe, and the dead-letter path must +// detect the resulting no-stream errors, drop the stale entry, and re-create +// the stream instead of failing every call until the client reconnects. +func TestPublishRecreatesDeletedStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wiped", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.wiped"))) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")}), + "publish must re-ensure the stream instead of failing on the stale memo") + + stream, serr := bd.js.Stream(ctx, streamNameFor("events.wiped")) + require.NoError(t, serr, "the stream must have been re-created") + assert.Equal(t, uint64(1), stream.CachedInfo().State.Msgs, "the retried publish must be stored") +} + +func TestSubscribeRecreatesDeletedStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wiped2", Subjects: []string{"created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.wiped2"))) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.wiped2.consumer", "events.wiped2", "created"), out) + + // The subscribe must have re-created the stream and its consumer; without + // the heal it returns fatal "stream not found" and nothing ever consumes. + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor("events.wiped2")) + return serr == nil && stream.CachedInfo().State.Consumers > 0 + }, 10*time.Second, 50*time.Millisecond, "subscribe must re-create the deleted stream") + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")})) + m := recv(t, out) + assert.Equal(t, "after", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) +} + +func TestDeadLetterRecoversDeletedDLQStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlsrc", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("p1")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("p2")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlsrc.consumer", "events.dlsrc", "job") + src.Options["DeadLetterAddress"] = "events.dlq.wiped" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + first := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, first.GetUuid()), "first dead-letter creates the DLQ stream") + + bd := p.getBrokerDetailsByIdentifier("test-client") + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.dlq.wiped"))) + + second := recv(t, out) + require.NotNil(t, p.DeadLetter(ctx, src, second.GetUuid()), + "the publish into the deleted DLQ stream must fail") + // The failed call dropped the stale memo, so this retry — the message is + // still in flight and resolvable, per the dead-letter failure contract — + // re-creates the DLQ stream and succeeds. + require.Nil(t, p.DeadLetter(ctx, src, second.GetUuid()), + "the dead-letter retry must re-create the DLQ stream") +} + func TestPrefetchMapsToMaxAckPending(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) From 89b860bb2bee2bb1eaa7479f87511acb773af147 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:45:21 -0400 Subject: [PATCH 28/77] fix: keep the original routing key on dead-letter copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RabbitMQ dead-letters a message under its original routing key unless the queue sets x-dead-letter-routing-key — which is what the DeadLetterSubject option maps to on the AMQP side. The natsjs DeadLetter republish only ever used DeadLetterSubject, so with no override every dead-lettered message landed on the dead-letter address's bare prefix: DLQ consumers binding by routing key missed them, and the copy no longer recorded where the message had been headed. Recover the original routing key from the delivered subject (the inverse of the publish mapping) and use it whenever DeadLetterSubject is unset. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 ++++-- .../provider/connectors/natsjs/helpers.go | 12 ++++++++ .../connectors/natsjs/helpers_test.go | 12 ++++++++ internal/provider/connectors/natsjs/natsjs.go | 12 +++++++- .../provider/connectors/natsjs/natsjs_test.go | 30 +++++++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 2ac834a..5d67816 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -184,8 +184,11 @@ default favors failover; RabbitMQ's equivalent consumer timeout defaults to dead-letter retry-queue idiom; JetStream increments the delivery count, which the connector surfaces back to the client as the `x-retry-count` header (see below). JetStream has no native dead-letter exchange, so `DeadLetter` -republishes the message to the configured dead-letter subject and then -`Term()`s it to stop redelivery. RabbitMQ performs that move broker-side; +republishes the message to the dead-letter address — under the message's +original routing key unless `DeadLetterSubject` overrides it, exactly as +RabbitMQ dead-letters under the original key unless +`x-dead-letter-routing-key` is set — and then `Term()`s it to stop +redelivery. RabbitMQ performs that move broker-side; here it is two proxy-side steps, so ordering is what protects the data: the original is terminated only after the DLQ publish succeeds. If the DLQ stream cannot be ensured or published to, `DeadLetter` returns an error and leaves @@ -322,7 +325,7 @@ amqp091 connector, so existing client sources validate unchanged: | `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). A warning is logged at subscribe time. | | `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. A warning is logged at subscribe time. | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | -| `DeadLetterSubject` | string | Routing key for dead-lettered messages. | +| `DeadLetterSubject` | string | Routing-key override for dead-lettered messages; when unset, the copy keeps the message's original routing key (RabbitMQ's default dead-letter behavior). | | `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | | `ConsumerGroup` | string | Durable consumer group name (stream sources; also names the durable that single-active instances coordinate through, and is required for a single-active stream source). | diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 1d4496c..5da3c6e 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -152,6 +152,18 @@ func translateWildcards(key string) string { return strings.Join(out, ".") } +// routingKeyFromSubject recovers the routing key a delivered message was +// published with by stripping the address's subject prefix — the inverse of +// publishSubjectFor for subjects the connector itself produced. The bare +// prefix (an empty routing key) and any subject not under the prefix (which +// a consumer's filter subjects make impossible) yield "". +func routingKeyFromSubject(addressName, subject string) string { + if rk, ok := strings.CutPrefix(subject, subjectPrefix(addressName)+"."); ok { + return rk + } + return "" +} + // streamSubjectsFor returns the subject set a stream captures for the given // address: the bare prefix (empty routing key) and everything under it. The // two are disjoint from each other ('>' needs at least one more token) and, diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 964c4a9..c9297a4 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -54,6 +54,18 @@ func TestPublishSubjectFor(t *testing.T) { assert.Equal(t, "events.orders.~.a_b", publishSubjectFor("events.orders", "a b")) } +func TestRoutingKeyFromSubject(t *testing.T) { + // inverse of publishSubjectFor + assert.Equal(t, "region.us.created", + routingKeyFromSubject("events.orders", "events.orders.~.region.us.created")) + // the bare prefix is the empty routing key + assert.Equal(t, "", routingKeyFromSubject("events.orders", "events.orders.~")) + // a subject outside the address's space yields no key + assert.Equal(t, "", routingKeyFromSubject("events.orders", "events.audit.~.x")) + // empty address: subjects live directly under the delimiter + assert.Equal(t, "a.b", routingKeyFromSubject("", "~.a.b")) +} + func TestStreamSubjectsFor(t *testing.T) { assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 43963cd..24c2534 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1009,8 +1009,18 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid dlqHeader.Set(retryCountHeaderName, strconv.FormatUint(md.NumDelivered-1, 10)) } } + // RabbitMQ dead-letters a message under its original routing key + // unless the queue overrides it (x-dead-letter-routing-key, which is + // what DeadLetterSubject maps to on the AMQP side). Preserve the + // original key the same way when no override is set, so DLQ consumers + // can bind by routing key and reprocessing tools can see where each + // message was originally headed. + dlSubject := opts["DeadLetterSubject"] + if dlSubject == "" { + dlSubject = routingKeyFromSubject(source.GetAddress().GetName(), m.Subject()) + } dlqMsg := &nats.Msg{ - Subject: publishSubjectFor(dla, opts["DeadLetterSubject"]), + Subject: publishSubjectFor(dla, dlSubject), Data: m.Data(), Header: dlqHeader, } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 1501ead..9207f53 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -242,6 +242,36 @@ func TestDeadLetter(t *testing.T) { "DLQ copy must carry the retry count at dead-letter time") } +// TestDeadLetterPreservesRoutingKey: when a source sets no DeadLetterSubject, +// the DLQ copy must keep the original message's routing key — RabbitMQ +// dead-letters under the original key unless x-dead-letter-routing-key +// overrides it, and DLQ consumers may bind by routing key. +func TestDeadLetterPreservesRoutingKey(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlrk", Subjects: []string{"region.us.created"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dlrk.consumer", "events.dlrk", "region.#") + src.Options["DeadLetterAddress"] = "events.dlrk.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlrk.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.Equal(t, "events.dlrk.dlq.~.region.us.created", raw.Subject, + "the DLQ copy must carry the original routing key under the DLA root") +} + // TestDeadLetterFailureKeepsMessage: when the dead-letter publish cannot happen // (here the DLQ address's subject space is already claimed by a foreign // stream, so ensuring its stream fails), DeadLetter must NOT Term the From 90e88085d10e8a321f5c014881c4f455cc983aa8 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:47:05 -0400 Subject: [PATCH 29/77] fix: report a durable source's own attached clients as its consumer count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SourceStats filled consumer_count from the stream's consumer total, and streams are shared per address root — so every queue bound to one exchange reported the consumers of all of them, and because the durable consumer itself counts whether or not any client is attached, the value could never distinguish an abandoned queue (zero attached clients) from a busy one. The amqp091 connector reports the queue's own consumer count. For a durable source, report the clients attached to its consumer instead: the consumer's open pull requests (NumWaiting). That is the pull-consumer equivalent of attached consumers — single-active standbys hold waiting pulls and are counted, like RabbitMQ counts inactive SAC consumers — with one caveat, documented in the design doc: a client working through a full pull buffer has no outstanding pull and can briefly read as zero. Sources without a durable consumer keep the stream-wide fallback. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 6 +++- internal/provider/connectors/natsjs/natsjs.go | 8 ++++++ .../provider/connectors/natsjs/natsjs_test.go | 28 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 5d67816..85e7164 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -354,7 +354,11 @@ amqp091 connector, so existing client sources validate unchanged: limits, so its depth keeps growing after consumers catch up, which would mislead anything using message count as queue length (e.g. consumer autoscaling). Sources without a durable consumer fall back to the stream - view. + view. A durable source's consumer count is likewise per source: the number + of clients with an open pull request on its consumer — a client working + through a full pull buffer can briefly read as zero — while sources + without a durable consumer report the stream-wide consumer count, which + spans every source on the address. - **Header filters are evaluated proxy-side.** NATS routes on subjects only, so a source's header `Filter`s cannot narrow what the server delivers: every message matching the source's subject filters is delivered to the diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 24c2534..c2a0fd4 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1145,6 +1145,14 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb ci := cons.CachedInfo() stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec + // The stream-wide consumer count set above spans every source on + // the address (streams are shared per address root), so it says + // nothing about THIS source — it cannot even distinguish zero + // attached clients from many, because the durable itself counts. + // The per-source equivalent of RabbitMQ's queue consumer count is + // the clients pulling this durable: report its open pull requests. + // (A client whose pull buffer is full can briefly read as zero.) + stats.ConsumerCount = int32(ci.NumWaiting) //nolint:gosec // Delivered.Consumer counts deliveries (redeliveries included, // like RabbitMQ's deliver rate). Ephemeral sources have no // consumer identity to sample here, so their rate stays zero. diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 9207f53..3fbe752 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1118,6 +1118,34 @@ func TestSourceStatsReportsConsumerBacklog(t *testing.T) { // ephemeral consumer is removed from the server as soon as its subscription // ends, instead of counting against the stream's consumer limit until the // inactivity threshold expires it. +// Streams are shared per address root, so the stream-wide consumer count +// spans every source on the address. A durable source's ConsumerCount must +// instead reflect the clients attached to ITS consumer — RabbitMQ reports the +// queue's own consumer count — or two queues on one exchange each report the +// other's consumers. +func TestSourceStatsConsumerCountIsPerSource(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + srcA := queueSource("events.cc.a", "events.cc", "created") + srcB := queueSource("events.cc.b", "events.cc", "created") + outA := make(chan *pb.Message, 1) + outB := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, srcA, outA) + go p.Subscribe(subCtx, srcB, outB) + + // Wait for both subscriptions' pulls to register, then check that each + // source counts only its own attached client. Pre-fix both report the + // stream-wide count of 2. + require.Eventually(t, func() bool { + return p.SourceStats(ctx, srcA).GetConsumerCount() == 1 && + p.SourceStats(ctx, srcB).GetConsumerCount() == 1 + }, 10*time.Second, 100*time.Millisecond, + "each durable source must report exactly its own attached client") +} + func TestEphemeralConsumerDeletedOnTeardown(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) From 8f682cc90d69d5d7ab4a0b66f9276e88f6dc93a2 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:48:39 -0400 Subject: [PATCH 30/77] fix: collapse consecutive '#' wildcards in binding patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An AMQP '#' matches zero or more words, so consecutive '#' tokens are equivalent to a single one — '#.#' is a legal spelling of 'everything'. The translation handled each token by position, so the first '#' of a trailing run landed in a non-terminal position and took the lossy narrow-to-'*' mapping: '#.#' became '*.>' (one-or-more) plus the zero-word variant of only the '>' tail, silently dropping the match on the empty routing key that a bare '#' keeps. Collapse runs of '#' into one before the position-based translation, so a trailing run gets the exact '>' tail mapping and only a '#' genuinely between words is narrowed. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 4 +++- .../provider/connectors/natsjs/helpers.go | 21 ++++++++++++++++--- .../connectors/natsjs/helpers_test.go | 12 +++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 85e7164..505a459 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -56,7 +56,9 @@ publishing routing key `filtered.x` to `events.orders` is indistinguishable from publishing `x` to `events.orders.filtered`. NATS subjects are stricter than AMQP routing keys, so each token is also -sanitized. In binding patterns (`translateWildcards`): a non-terminal `#` +sanitized. In binding patterns (`translateWildcards`): runs of consecutive +`#` collapse into one first (they are equivalent in AMQP, and `#.#` must +keep the zero-word match its trailing `#` provides), a non-terminal `#` becomes `*` (NATS `>` is tail-only), illegal characters (space, tab, `*`, `>`) inside a literal token become `_`, and empty tokens (from `a..b` or a trailing dot) are dropped. A pattern whose trailing `#` became `>` also gets diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 5da3c6e..4594d55 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -128,10 +128,27 @@ var tokenSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_" // would otherwise be empty — and thus invalid — tokens) // other -> illegal chars (space/tab/'*'/'>') replaced with '_' // +// Runs of consecutive '#' are collapsed into one before translation — they +// are equivalent in AMQP ('#' matches zero or more words) — so "#.#" gets +// the exact tail mapping instead of a first-'#' narrowed to '*'. +// // The result is always a valid NATS subject fragment (possibly empty if every // token was empty; filterSubjectsFor handles that). func translateWildcards(key string) string { - tokens := strings.Split(key, ".") + raw := strings.Split(key, ".") + // Drop empty tokens, and collapse runs of consecutive '#' into one: AMQP's + // '#' matches zero or more words, so adjacent '#'s are equivalent to a + // single one ("a.#.#" ≡ "a.#"). Collapsing has to happen before the + // position-based translation below, or a trailing run like "#.#" would put + // its first '#' in a non-terminal position and take the lossy '*' mapping — + // losing the zero-word match a lone trailing '#' keeps. + tokens := make([]string, 0, len(raw)) + for _, t := range raw { + if t == "" || (t == "#" && len(tokens) > 0 && tokens[len(tokens)-1] == "#") { + continue + } + tokens = append(tokens, t) + } out := make([]string, 0, len(tokens)) for i, t := range tokens { switch t { @@ -143,8 +160,6 @@ func translateWildcards(key string) string { } else { out = append(out, "*") } - case "": - // drop empty token default: out = append(out, tokenSanitizer.Replace(t)) } diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index c9297a4..6e5123e 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -137,6 +137,18 @@ func TestFilterSubjectsFor(t *testing.T) { assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, filterSubjectsFor(src("#"))) + // consecutive '#'s are one '#' in AMQP (each matches zero or more words): + // "#.#" must still match the empty routing key, and "a.#.#" must not lose + // zero-or-more to the non-terminal-'#' narrowing + assert.Equal(t, + []string{"events.orders.~", "events.orders.~.>"}, + filterSubjectsFor(src("#.#"))) + assert.Equal(t, + []string{"events.orders.~.a", "events.orders.~.a.>"}, + filterSubjectsFor(src("a.#.#"))) + assert.Equal(t, + []string{"events.orders.~.*.b"}, + filterSubjectsFor(src("#.#.b"))) // duplicates collapse assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, From 5f44bb177bf5d2d40273cf327ddbe54dabafe133 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 14:49:15 -0400 Subject: [PATCH 31/77] docs: state that transient sources deliver per subscriber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two clients subscribing with the same transient (auto-delete / exclusive / TEMPORARY) source name each get their own ephemeral consumer, so both receive every message — where consumers of one auto-delete AMQP queue would compete for its messages, and an exclusive AMQP queue would reject the second consumer. Clients conventionally generate unique names for transient sources, so this stays a documented limitation rather than a behavior change; the parity fix (a shared named consumer with an inactivity threshold, plus rejecting a second exclusive subscriber) is sketched with it. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 505a459..18fa311 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -299,7 +299,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | RabbitMQ / Arke feature | Disposition | Notes | | --- | --- | --- | | Topic routing + wildcard bindings | Native | NATS subjects + `*`/`>`; `#`->`>`, `*`->`*`. | -| Per-subscriber ephemeral queue | Native | Ephemeral consumer with inactivity threshold. | +| Per-subscriber ephemeral queue | Native | Ephemeral consumer with inactivity threshold (per subscriber — see limitations). | | Durable work queue (reconnect resumes backlog) | Native | Durable consumer for non-transient / consumer-group / single-active sources. | | Publish confirms | Native | `js.PublishMsg` returns a `PubAck`. | | Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | @@ -374,6 +374,16 @@ amqp091 connector, so existing client sources validate unchanged: fed by the connector; there is no advisory-driven re-consumption. A failed DLQ publish fails the dead-letter call — the message stays in flight and is redelivered — rather than dropping the message. +- **Transient sources are per-subscriber.** A transient (auto-delete / + exclusive / TEMPORARY) source maps to an ephemeral consumer created per + subscription, so two clients subscribing with the same transient source + name each receive every message. Consumers of one auto-delete AMQP queue + would instead compete for its messages, and an exclusive AMQP queue would + reject the second consumer outright. In practice clients generate unique + names for transient sources (the usual exclusive-queue idiom), which is + why this has not warranted the parity fix: a shared named consumer with an + inactivity threshold standing in for the queue's expiry, plus rejecting a + second subscriber when the source is exclusive. - **Connection authentication.** The connector supports user/password and TLS today; NKEYs / JWT auth are a natural follow-up for production deployments. From 04c8bb83a17ab5b071cd3dffd4e4d27ed7640eab Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:05:17 -0400 Subject: [PATCH 32/77] fix: derive collision-free stream and durable names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stream name (and durable consumer name, which shares the helper) was derived by replacing '.' with '_' in the sanitized address root. That replacement is not injective: addresses "a.b" and "a_b" both mapped to stream "arke_a_b", so whichever was ensured second either failed its subscribes (consumer filters outside the stream's subjects) or — via the stale-stream self-heal — updated the shared stream's subjects to its own address, breaking the other address's publishes and consumers in a ping-pong. Source names "grp.a" and "grp_a" likewise collapsed onto one durable, silently turning two independent queues into competing consumers. Roots containing '/' or '\' (legal in subjects, rejected in JetStream names) produced names the server refuses outright. Names free of those characters keep the readable historical form ("events.orders" -> "arke_events_orders"). A root containing '_' or a character JetStream rejects in names now takes an escaped form under the disjoint "arke-" prefix, in which every '_' starts a two-character escape code, so decoding is unambiguous and distinct roots always yield distinct names across both forms. Such names previously collided or were invalid, so no working deployment's stream or durable names change. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 34 ++++++++--- .../provider/connectors/natsjs/helpers.go | 55 ++++++++++++++++-- .../connectors/natsjs/helpers_test.go | 37 +++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 56 +++++++++++++++++++ 4 files changed, 168 insertions(+), 14 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 18fa311..38ddba5 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -42,6 +42,19 @@ an empty routing key maps to the bare prefix `events.orders.~`. A JetStream stream named `arke_
` captures `
.~` and `
.~.>`, and each consumer filters on the mapped source subjects. +Stream names (and durable consumer names, which JetStream validates the same +way) may not contain `.`, whitespace, `*`, `>`, `/` or `\`, so they are +derived from the address (or source / consumer-group) name by swapping dots +for underscores: `events.orders` becomes `arke_events_orders`. That +replacement alone is ambiguous the moment a name contains a literal `_` — +`a.b` and `a_b` would both read `arke_a_b`, and two addresses colliding onto +one stream name silently reconfigure each other's stream — so such names +take an escaped form instead, under the disjoint `arke-` prefix, in which +every `_` starts a two-character escape code (`_d` for `.`, `_u` for `_`, +and codes for the other characters JetStream rejects in names). Distinct +names therefore always yield distinct streams and durables, while names free +of underscores keep the readable historical form. + The `~` delimiter is what keeps distinct addresses' streams disjoint, which JetStream requires: two subjects may belong to at most one stream. Address names are themselves dotted, so two addresses can sit in a prefix @@ -79,14 +92,19 @@ filters to the widest set before creating the consumer; the surviving wildcard filter matches everything the dropped filters did. Intersecting patterns where neither contains the other are kept as-is. -This subject scheme is part of the connector's persistence contract: -retained messages are stored under these subjects for the life of the -stream's retention limits, so any future change to the encoding is a -breaking change for deployed data (`CreateOrUpdateStream` moves the stream's -captured subjects forward, after which messages stored under the previous -encoding no longer match any consumer filter and age out). Treat this layout -as canonical: external tooling that reads or writes JetStream subjects -directly must use the same mapping, including the `~` delimiter. +This subject scheme — and the stream/durable name encoding above — is part +of the connector's persistence contract: retained messages are stored under +these subjects for the life of the stream's retention limits, so any future +change to the encoding is a breaking change for deployed data +(`CreateOrUpdateStream` moves the stream's captured subjects forward, after +which messages stored under the previous encoding no longer match any +consumer filter and age out; a renamed stream or durable simply strands the +old one and its state). Treat this layout as canonical: external tooling +that reads or writes JetStream subjects directly must use the same mapping, +including the `~` delimiter. Deployments whose address or source names +contain `_`, `/` or `\` predating the escaped name form had those names +colliding or rejected outright, so the escaped form changes only names that +were already broken. AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 4594d55..e29f40c 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -78,11 +78,58 @@ func subjectPrefix(addressName string) string { return addressDelim } -// streamNameFor derives a JetStream-legal stream name from an address name. -// Stream names may not contain '.', ' ', '*' or '>'. Derived from the -// sanitized root so every address that shares a root shares a stream. +// nameEscapes encodes, inside the escaped name form (see streamNameFor), the +// characters JetStream rejects in stream/consumer names plus '_' itself. +// Every code is two characters starting with '_' and no literal character +// maps to '_', so an escaped name decodes unambiguously left to right — +// which is what makes the encoding injective. +var nameEscapes = map[rune]string{ + '.': "_d", + '_': "_u", + '/': "_s", + '\\': "_b", + '\r': "_r", + '\n': "_n", + '\f': "_f", +} + +// nameEscapeTriggers are the characters that force a root into the escaped +// name form: '_' because the plain form could not tell it apart from an +// encoded '.', and the rest because JetStream rejects them in names outright +// (they are legal in subjects, so sanitization keeps them in the root). +const nameEscapeTriggers = "_/\\\r\n\f" + +// streamNameFor derives a JetStream-legal name from an address (or consumer +// group / source) name; durable consumer names use it too, as JetStream +// validates both identically. Names may not contain whitespace, '.', '*', +// '>', '/' or '\'. Derived from the sanitized root so every address that +// shares a root shares a stream. +// +// The common case — tokens free of '_' and of characters JetStream rejects +// in names — keeps the readable historical form, dots swapped for +// underscores ("events.orders" -> "arke_events_orders"). That replacement is +// only unambiguous while no token contains '_' of its own: "a.b" and "a_b" +// would otherwise both read "arke_a_b", and two addresses colliding onto one +// stream name silently reconfigure each other's stream (each re-ensure +// flips the stream's subjects to its own address). Such roots instead take +// an escaped form under the disjoint "arke-" prefix, where every '_' starts +// a two-character escape code (nameEscapes), so distinct roots always yield +// distinct names across both forms. func streamNameFor(addressName string) string { - return "arke_" + strings.ReplaceAll(addressRoot(addressName), ".", "_") + root := addressRoot(addressName) + if !strings.ContainsAny(root, nameEscapeTriggers) { + return "arke_" + strings.ReplaceAll(root, ".", "_") + } + var b strings.Builder + b.WriteString("arke-") + for _, r := range root { + if esc, ok := nameEscapes[r]; ok { + b.WriteString(esc) + } else { + b.WriteRune(r) + } + } + return b.String() } // publishSubjectFor maps an address + routing key onto the concrete subject a diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 6e5123e..d40b7a2 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -14,9 +14,37 @@ import ( func TestStreamNameFor(t *testing.T) { assert.Equal(t, "arke_events_orders", streamNameFor("events.orders")) assert.Equal(t, "arke_events_audit", streamNameFor("events.audit")) - // illegal subject chars are sanitized - assert.Equal(t, "arke_a_b_c", streamNameFor("a.b*c")) assert.Equal(t, "arke_", streamNameFor("")) + + // Names whose tokens contain '_' (directly, or via sanitization of an + // illegal subject char) take the escaped "arke-" form so they cannot + // collide with a dotted sibling: under the plain dots-to-underscores + // replacement "a.b" and "a_b" would both read "arke_a_b" and clobber each + // other's stream. + assert.Equal(t, "arke_a_b", streamNameFor("a.b")) + assert.Equal(t, "arke-a_ub", streamNameFor("a_b")) + assert.Equal(t, "arke-a_ub_dc", streamNameFor("a_b.c")) + // sanitized chars ('*' -> '_') route through the escaped form too + assert.Equal(t, "arke-a_db_uc", streamNameFor("a.b*c")) + // characters JetStream rejects in names but allows in subjects are escaped + assert.Equal(t, "arke-a_sb", streamNameFor("a/b")) + assert.Equal(t, "arke-a_bb", streamNameFor(`a\b`)) + + // The mapping must be injective: distinct address roots may never share a + // stream name, or the two addresses silently reconfigure each other's + // stream. These are the collision pairs of the plain replacement. + collisionProne := []string{"a.b", "a_b", "a.b.c", "a_b.c", "a.b_c", "a_b_c", "a/b", `a\b`} + seen := map[string]string{} + for _, addr := range collisionProne { + name := streamNameFor(addr) + if prev, ok := seen[name]; ok { + t.Errorf("streamNameFor collision: %q and %q both map to %q", prev, addr, name) + } + seen[name] = addr + assert.NotContains(t, name, ".", "stream name must be JetStream-legal") + assert.NotContains(t, name, "/", "stream name must be JetStream-legal") + assert.NotContains(t, name, `\`, "stream name must be JetStream-legal") + } } func TestAddressRoot(t *testing.T) { @@ -257,6 +285,11 @@ func TestDurableName(t *testing.T) { st2 := q("st2") st2.Type = pb.Source_STREAM assert.Equal(t, "", durableName(st2)) + + // durable names inherit streamNameFor's injectivity: source names "grp.a" + // and "grp_a" must not share a durable, or the two sources silently + // become competing consumers on one queue instead of independent queues + assert.NotEqual(t, durableName(q("grp.a")), durableName(q("grp_a"))) } func TestEvaluateFilters(t *testing.T) { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 3fbe752..a808b69 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -798,6 +798,62 @@ func TestPrefixAddressesCoexist(t *testing.T) { assert.True(t, got["to-parent"] && got["still-to-parent"], "got %v", got) } +// TestUnderscoreAddressesCoexist: address names "events.a" and "events_a" are +// distinct AMQP exchanges, but a naive dots-to-underscores stream name maps +// both onto one JetStream stream, whose config each address's re-ensure then +// flips to its own subjects — each breaking the other. The name mapping must +// keep them on separate streams. +func TestUnderscoreAddressesCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + dotted := &pb.Address{Name: "events.a", Subjects: []string{"k"}} + scored := &pb.Address{Name: "events_a", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: dotted, Body: []byte("to-dotted")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: scored, Body: []byte("to-scored")}), + "publishing to events_a after events.a must not hijack its stream") + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + for addr, want := range map[string]string{"events.a": "to-dotted", "events_a": "to-scored"} { + out := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource(addr+".consumer", addr, "k"), out) + m := recv(t, out) + assert.Equal(t, want, string(m.GetBody()), "consumer on %q sees its own message", addr) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery on %q: %q", addr, extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + } +} + +// TestUnderscoreSourceNamesStayIndependent: two queues named "grp.a" and +// "grp_a" bound to the same address are independent queues — each receives +// every message — but a naive name mapping collapses them onto one durable, +// turning them into competing consumers that split the traffic. +func TestUnderscoreSourceNamesStayIndependent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.split", Subjects: []string{"k"}} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + outDot := make(chan *pb.Message, 2) + outScore := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource("grp.a", "events.split", "k"), outDot) + go p.Subscribe(subCtx, queueSource("grp_a", "events.split", "k"), outScore) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("fan-out")})) + + for name, out := range map[string]<-chan *pb.Message{"grp.a": outDot, "grp_a": outScore} { + m := recv(t, out) + assert.Equal(t, "fan-out", string(m.GetBody()), "queue %q gets its own copy", name) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } +} + // streamSource builds a stream-style source positioned at the given RabbitMQ // Streams offset ("first"/"next"/...). It is the offset-parameterized sibling // of queueSource. From 0579eca9134530ff6c8d1698c7293699e63fa733 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:07:03 -0400 Subject: [PATCH 33/77] fix: track each subscription's consume context separately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live subscriptions were registered in consumeContexts under the source name, but one connection can hold several subscriptions for the same name — consumer groups (and single-active groups in particular) share a source name and differ only by their ConsumerGroup option. The second Add overwrote the first, so Stats undercounted the connection's streams, Disconnect could only Stop the last-registered context, and whichever subscription tore down first deleted the map entry the survivor still needed. Each subscription now registers under a unique key (source name plus a per-connection serial), so entries never collide; teardown removes exactly its own. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 15 ++++-- .../provider/connectors/natsjs/natsjs_test.go | 50 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index c2a0fd4..ec9ad19 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -102,8 +102,16 @@ type natsBrokerDetails struct { activeMessages *util.ConcurrentMap // knownStreams memoizes streams we have already ensured. knownStreams *util.ConcurrentMap - // consumeContexts maps source name -> jetstream.ConsumeContext for teardown. + // consumeContexts holds each live subscription's jetstream.ConsumeContext + // so Disconnect can stop them and Stats can count them. Keyed per + // subscription (source name plus a serial from subscriptionSeq), NOT per + // source name: one connection may legitimately subscribe the same source + // name more than once — single-active groups share a name and differ only + // by ConsumerGroup — and a name key would make those overwrite each other, + // undercounting Stats and letting one teardown drop the other's entry. consumeContexts *util.ConcurrentMap + // subscriptionSeq disambiguates consumeContexts keys. + subscriptionSeq atomic.Uint64 // rates derives SourceStats publish/deliver rates from JetStream counters. rates *rateTracker @@ -710,7 +718,8 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} } - bd.consumeContexts.Add(source.GetName(), cc) + subKey := fmt.Sprintf("%s#%d", source.GetName(), bd.subscriptionSeq.Add(1)) + bd.consumeContexts.Add(subKey, cc) defer func() { cc.Stop() // Stop is asynchronous: wait (bounded) for the consume loop to finish @@ -723,7 +732,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c case <-time.After(ephemeralDeleteTimeout): } _ = bd.nc.FlushTimeout(ephemeralDeleteTimeout) - bd.consumeContexts.Delete(source.GetName()) + bd.consumeContexts.Delete(subKey) releaseInFlight(bd, streamName, cons.CachedInfo().Name, consCfg.Durable != "") if consCfg.Durable == "" { deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index a808b69..5785b19 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -854,6 +854,56 @@ func TestUnderscoreSourceNamesStayIndependent(t *testing.T) { } } +// TestSameSourceNameSubscriptionsTrackedSeparately: consumer groups share one +// source name and differ only by ConsumerGroup, so a single connection can +// hold several live subscriptions for the same name. Each must keep its own +// bookkeeping entry: keyed by name alone, the second Add overwrites the +// first, Stats undercounts the connection's streams, and the first teardown +// deletes the survivor's entry (so Disconnect would no longer Stop it). +func TestSameSourceNameSubscriptionsTrackedSeparately(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := func(cg string) *pb.Source { + return &pb.Source{ + Name: "events.shared.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.shared", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first", "ConsumerGroup": cg}, + } + } + + outA := make(chan *pb.Message, 2) + outB := make(chan *pb.Message, 2) + ctxA, cancelA := context.WithCancel(ctx) + ctxB, cancelB := context.WithCancel(ctx) + defer cancelB() + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src("grp.a"), outA) }() + go p.Subscribe(ctxB, src("grp.b"), outB) + + // Each group's durable receives the message, proving both subscriptions + // are live and consuming. + addr := &pb.Address{Name: "events.shared", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + require.Nil(t, p.Ack(ctx, recv(t, outB).GetUuid())) + + require.Eventually(t, func() bool { + stats := p.Stats() + return len(stats.Clients) == 1 && stats.Clients[0].Streams == 2 + }, 5*time.Second, 20*time.Millisecond, + "two live subscriptions sharing a source name must both be tracked") + + // Ending one subscription removes only its own entry. + cancelA() + require.Nil(t, <-errA) + stats := p.Stats() + require.Len(t, stats.Clients, 1) + assert.Equal(t, 1, stats.Clients[0].Streams, + "the sibling's teardown must not drop the survivor's entry") +} + // streamSource builds a stream-style source positioned at the given RabbitMQ // Streams offset ("first"/"next"/...). It is the offset-parameterized sibling // of queueSource. From f1cced8e38e3da7d8279ee2423040fdeedc82aa0 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:08:09 -0400 Subject: [PATCH 34/77] fix: include the durable identity in publish-rate sample keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publish-rate samples were keyed by stream and source name. Consumer groups share one source name and differ only by their ConsumerGroup option — so two groups polling stats on the same address shared a sample key, and each poll reset the other group's baseline window, reporting the rate over the gap since the sibling's poll instead of the poller's own interval. The deliver-rate key already carried the durable name; the publish-rate key now does too. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 17 +++++---- .../provider/connectors/natsjs/natsjs_test.go | 38 +++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index ec9ad19..ad8d4e0 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1131,15 +1131,18 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb stats.MessageCount = int64(info.State.Msgs) //nolint:gosec stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec stats.LastOffset = int64(info.State.LastSeq) //nolint:gosec + durable := durableName(source) // JetStream exposes no rates, only counters; sample them between calls // (see rateTracker). The stream's LastSeq counts every publish to the // address root, so the publish rate is per address, not per binding. - // The sample key includes the polling source: the counters are shared, - // but each source polls on its own cadence, and a shared key would make - // every observation window the gap since whichever source polled last — - // with several sources on one address the later polls of a cycle would - // report rates over milliseconds instead of the poll interval. - stats.PublishRate = bd.rates.observe("pub/"+streamName+"/"+source.GetName(), now, info.State.LastSeq) + // The sample key includes the polling identity — durable and source name, + // since consumer groups share a source name and differ only by their + // durable: the counters are shared, but each poller runs on its own + // cadence, and a shared key would make every observation window the gap + // since whichever poller came last — with several pollers on one address + // the later polls of a cycle would report rates over milliseconds instead + // of the poll interval. + stats.PublishRate = bd.rates.observe("pub/"+streamName+"/"+durable+"/"+source.GetName(), now, info.State.LastSeq) // For a source with a durable consumer, report that consumer's actual // backlog (undelivered + delivered-but-unacked) instead of the stream // depth: the stream retains acked messages under its retention limits, so @@ -1147,7 +1150,7 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb // matches the amqp091 connector, which reports the queue's ready+unacked // count — the number consumer-scaling logic wants. Without a durable // consumer (or before it exists), fall back to the stream view. - if durable := durableName(source); durable != "" { + if durable != "" { if cons, cerr := stream.Consumer(ctx, durable); cerr == nil { // Consumer() fetched fresh info to return the handle; reuse it // instead of paying a second round-trip on every stats poll. diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 5785b19..8f5577b 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1668,6 +1668,44 @@ func TestSourceStatsRatesIndependentPerSource(t *testing.T) { "another source's poll must not reset this source's rate window") } +// TestSourceStatsRatesIndependentPerGroup: consumer groups share one source +// name and differ only by ConsumerGroup, so a name-keyed publish-rate sample +// makes the groups reset each other's baseline window exactly like the +// per-source case above — the sample key must carry the durable identity too. +func TestSourceStatsRatesIndependentPerGroup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.grpsplit", Subjects: []string{"e"}} + src := func(cg string) *pb.Source { + return &pb.Source{ + Name: "events.grpsplit.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.grpsplit", Subjects: []string{"e"}}, + Options: map[string]string{"ConsumerGroup": cg}, + } + } + srcA, srcB := src("grp.a"), src("grp.b") + + // Create the stream and baseline group A. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("x")})) + require.Nil(t, p.SourceStats(ctx, srcA).GetError()) + + for i := 0; i < 5; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("y")})) + } + + // Group B polls (its own baseline) between A's polls. A's next poll must + // still see the 5 publishes inside its own window. + require.Nil(t, p.SourceStats(ctx, srcB).GetError()) + time.Sleep(20 * time.Millisecond) + + stats := p.SourceStats(ctx, srcA) + require.Nil(t, stats.GetError()) + assert.Positive(t, stats.GetPublishRate(), + "another group's poll must not reset this group's rate window") +} + func TestUnappliedSourceOptions(t *testing.T) { src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } From fd505086501a605459864374e5cb65d9bc13ba73 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:09:06 -0400 Subject: [PATCH 35/77] docs: note the gRPC server raises prefetch below 1 before providers see it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connector maps prefetch 0 (AMQP unlimited) to JetStream's unlimited MaxAckPending, but Arke's gRPC server defaults any prefetch below 1 to 1 (SetSourceDefaults) before calling Subscribe, so a subscribe arriving through the server never reaches that branch. State that where the mapping is described — code comment and parity table — so nobody reads the unlimited mapping as reachable end-to-end through the proxy. The mapping stays: it keeps the provider contract correct for direct in-process use and for any future change to the server-side defaulting, which is shared across connectors and deliberately conservative. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 2 +- internal/provider/connectors/natsjs/natsjs.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 38ddba5..dbfb51f 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -323,7 +323,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Message dedup (`publish_id` + `publisher_name`) | Native | `Nats-Msg-Id` + stream `Duplicates` window. | | Streams: offsets, start position | Native | JetStream is a log; `DeliverPolicy` maps `Offset`. | | Single active consumer | Native | Pinned-client priority group on the durable (nats-server 2.11+); standby takes over within `NATSJS_SAC_PINNED_TTL`. | -| Prefetch / QoS | Native | `MaxAckPending`; prefetch 0 (AMQP "unlimited") maps to unlimited (-1). | +| Prefetch / QoS | Native | `MaxAckPending`; prefetch 0 (AMQP "unlimited") maps to unlimited (-1). The Arke gRPC server raises a prefetch below 1 to 1 before any provider sees it, so the unlimited mapping applies to direct provider use. | | HA / quorum queues | Native | JetStream R3 (Raft) via `NATSJS_STREAM_REPLICAS`. | | Delayed retry (per-msg TTL + DLX idiom) | Proxy -> Native | Replaced with `NakWithDelay`. | | Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index ad8d4e0..8a24dbb 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -598,7 +598,12 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // AMQP prefetch 0 means unlimited (amqp091 leaves the channel default in // that case rather than calling SetPrefetch), so map it to JetStream's // explicit unlimited (-1) — not to the server default of 1000, and - // certainly not to 1, which would silently serialize the consumer. + // certainly not to 1, which would silently serialize the consumer. Note + // that Arke's gRPC server raises a prefetch below 1 to 1 before any + // provider sees the source (SetSourceDefaults, internal/server), so a + // subscribe arriving through the server never takes this branch; it keeps + // the provider contract honest for direct (in-process) users and for any + // future change to the server-side defaulting. prefetch := int(source.GetPrefetchCount()) if prefetch <= 0 { prefetch = -1 From 11c79fb4d016b0a56e681050977a6382c0b05c26 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:10:59 -0400 Subject: [PATCH 36/77] test: type the stream-offset test sources as Source_STREAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streamSource helper claimed to build a stream-style source but set Source_QUEUE, so the offset/replay suite exercised durable queue behavior rather than the Source_STREAM branches of the consumer-kind selection. The helper now builds a real stream source carrying a ConsumerGroup — the durable identity a stream source coordinates through — so the offset tests cover the stream-with-group path, and a new test pins the group-less form: subscribers of a STREAM source without a ConsumerGroup are independent ephemeral readers, each positioned by its own Offset and each seeing every message, which is deliberate parity (stream consumers read a shared log and never compete). The design doc's ephemeral bullet now names that case. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 5 +- .../provider/connectors/natsjs/natsjs_test.go | 54 +++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index dbfb51f..8c5205f 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -123,7 +123,10 @@ The connector chooses the consumer kind from the source and any `SingleActiveConsumer`. - Ephemeral consumers auto-expire after an inactivity threshold. Used for auto-delete / exclusive / `TEMPORARY` sources, which clients commonly use - for per-instance, transient subscriptions. They are also deleted eagerly + for per-instance, transient subscriptions — and for `STREAM` sources + without a `ConsumerGroup`, whose subscribers are independent readers of + the shared log, each positioned by its own `Offset` (RabbitMQ stream + consumers do not compete). They are also deleted eagerly when their subscription ends (or never starts — a declare-only call, or a failure to begin consuming), so the threshold only has to cover unclean exits and churning transient clients cannot accumulate dead consumers diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 8f5577b..f08b711 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -904,15 +904,59 @@ func TestSameSourceNameSubscriptionsTrackedSeparately(t *testing.T) { "the sibling's teardown must not drop the survivor's entry") } -// streamSource builds a stream-style source positioned at the given RabbitMQ -// Streams offset ("first"/"next"/...). It is the offset-parameterized sibling -// of queueSource. +// streamSource builds a stream-typed source (Source_STREAM) positioned at the +// given RabbitMQ Streams offset ("first"/"next"/...). It carries a +// ConsumerGroup, derived from the source name, because the group is a stream +// source's durable identity — each test consumer keeps its own durable. See +// TestStreamSourceWithoutGroupReadsIndependently for the group-less +// (ephemeral) form. func streamSource(name, addr, offset string, subjects ...string) *pb.Source { return &pb.Source{ Name: name, - Type: pb.Source_QUEUE, + Type: pb.Source_STREAM, Address: &pb.Address{Name: addr, Subjects: subjects}, - Options: map[string]string{"Offset": offset}, + Options: map[string]string{"Offset": offset, "ConsumerGroup": name + ".grp"}, + } +} + +// TestStreamSourceWithoutGroupReadsIndependently: a STREAM source with no +// ConsumerGroup has no durable identity, so every subscriber is an +// independent ephemeral reader positioned by its own Offset and sees every +// message. That is deliberate RabbitMQ parity — stream consumers read a +// shared log and never compete (unlike queue consumers) — distinct from the +// transient-source fan-out documented as a known limitation. +func TestStreamSourceWithoutGroupReadsIndependently(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.reader", Subjects: []string{"e"}} + const n = 3 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("m%d", i))})) + } + + src := func() *pb.Source { + return &pb.Source{ + Name: "events.reader.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.reader", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "first"}, + } + } + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + outA := make(chan *pb.Message, n) + outB := make(chan *pb.Message, n) + go p.Subscribe(subCtx, src(), outA) + go p.Subscribe(subCtx, src(), outB) + for _, out := range []<-chan *pb.Message{outA, outB} { + for i := 0; i < n; i++ { + m := recv(t, out) + assert.Equal(t, fmt.Sprintf("m%d", i), string(m.GetBody()), + "each reader replays the full log in order") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } } } From 525b442392011ddf5379ad487a13cce6b9b87385 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 18:29:41 -0400 Subject: [PATCH 37/77] fix: tighten natsjs edge-case parity Three small parity edges could change broker-visible behavior. Dedup now only sets Nats-Msg-Id for the valid publish_id + publisher_name pair and rejects a publish id without a publisher name, matching the stream-publish contract. Dead-letter now treats a present but empty DeadLetterAddress as an error that leaves the original in flight, so the server fallback can nack and redeliver instead of terming a message with no DLQ copy. Direct address bindings now map subjects as exact routing keys, so # and * are literals for Address_QUEUE rather than topic wildcards. Regression tests cover each case, and the design doc now records the corrected semantics. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 26 ++-- .../provider/connectors/natsjs/helpers.go | 22 ++++ .../connectors/natsjs/helpers_test.go | 17 +++ internal/provider/connectors/natsjs/natsjs.go | 112 ++++++++++-------- .../provider/connectors/natsjs/natsjs_test.go | 91 ++++++++++++++ 5 files changed, 207 insertions(+), 61 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 8c5205f..849b3b4 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -82,6 +82,10 @@ meaning only in bindings, and NATS forbids wildcard tokens in published subjects — so wildcard characters are sanitized to `_` like any other illegal character. In address names, empty tokens and tokens equal to `~` are replaced with `_` rather than dropped, so distinct names keep distinct roots. +For `Address_QUEUE` (AMQP direct-exchange parity), binding subjects are exact +routing keys rather than topic patterns: `#` and `*` are literals and are +sanitized the same way a published routing key is sanitized, instead of +matching as wildcards. A redundant binding set — a wildcard binding alongside a specific key it already covers, such as `orders.#` with `orders.created` — is legal in AMQP, @@ -217,12 +221,14 @@ original is terminated only after the DLQ publish succeeds. If the DLQ stream cannot be ensured or published to, `DeadLetter` returns an error and leaves the message ack-pending — the server then falls back to a nack, so the message is redelivered and dead-lettering is retried instead of the message -being lost. The DLQ copy carries a `Nats-Msg-Id` derived from the original's -stream sequence, so a retried dead-letter of the same message deduplicates in -the DLQ within its dedup window, and the `x-retry-count` the consumer saw -when it gave the message up — RabbitMQ's broker-side move preserves the -death trail in `x-death`, and the retry count is this connector's equivalent -(a plain republish would lose it). +being lost. A present but empty `DeadLetterAddress` is treated the same way: +the connector returns an error and leaves the original in flight rather than +terminating it without a DLQ publish. The DLQ copy carries a `Nats-Msg-Id` +derived from the original's stream sequence, so a retried dead-letter of the +same message deduplicates in the DLQ within its dedup window, and the +`x-retry-count` the consumer saw when it gave the message up — RabbitMQ's +broker-side move preserves the death trail in `x-death`, and the retry count +is this connector's equivalent (a plain republish would lose it). ## Retry-count header @@ -235,9 +241,11 @@ equivalent. ## Deduplication Publish-side dedup maps onto JetStream's `Nats-Msg-Id` plus the stream -`Duplicates` window. When a message carries a publisher name and/or publish id, -the connector sets `Nats-Msg-Id` so re-publishes within the window are -collapsed. +`Duplicates` window. When a message carries a positive publish id, it must +also carry a publisher name (matching the stream-publish contract of the AMQP +connector); the connector sets `Nats-Msg-Id` from that pair so re-publishes +within the window are collapsed. A publisher name by itself does not enable +deduplication. ## Retention: work-queue vs append-log diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index e29f40c..3329172 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -241,6 +241,10 @@ func streamSubjectsFor(addressName string) []string { // AMQP '#' matches zero or more words while NATS '>' matches one or more, so // binding "a.#" must match routing key "a" too. func filterSubjectsFor(source *pb.Source) []string { + if source.GetAddress().GetType() == pb.Address_QUEUE { + return directFilterSubjectsFor(source) + } + prefix := subjectPrefix(source.GetAddress().GetName()) pats := source.GetAddress().GetSubjects() if len(pats) == 0 { @@ -294,6 +298,24 @@ func filterSubjectsFor(source *pb.Source) []string { return kept } +func directFilterSubjectsFor(source *pb.Source) []string { + prefix := subjectPrefix(source.GetAddress().GetName()) + pats := source.GetAddress().GetSubjects() + if len(pats) == 0 { + return []string{prefix} + } + out := make([]string, 0, len(pats)) + seen := make(map[string]bool, len(pats)) + for _, p := range pats { + s := publishSubjectFor(source.GetAddress().GetName(), p) + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + // subjectSubsumes reports whether every subject matched by narrow is also // matched by wide (both may contain NATS wildcards): a wide '>' covers any // one or more remaining tokens, and a wide '*' covers a literal token or diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index d40b7a2..b7932f4 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -200,6 +200,23 @@ func TestFilterSubjectsFor(t *testing.T) { filterSubjectsFor(src("*.b", "a.*"))) } +func TestDirectFilterSubjectsAreExact(t *testing.T) { + src := func(subjects ...string) *pb.Source { + return &pb.Source{Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: subjects}} + } + + assert.Equal(t, + []string{"events.direct.~._"}, + filterSubjectsFor(src("#")), + "direct bindings treat # as a literal routing key, not a wildcard") + assert.Equal(t, + []string{"events.direct.~.created", "events.direct.~.region.us"}, + filterSubjectsFor(src("created", "region.us", "created"))) + assert.Equal(t, + []string{"events.direct.~"}, + filterSubjectsFor(src())) +} + func TestSubjectSubsumes(t *testing.T) { cover := func(wide, narrow string) bool { return subjectSubsumes(wide, narrow) } diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 8a24dbb..c5d97ea 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -516,7 +516,10 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, } var pubOpts []jetstream.PublishOpt // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). - if msg.GetPublisherName() != "" || msg.GetPublishId() != 0 { + if msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { + return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} + } + if msg.GetPublishId() > 0 { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) } _, perr := bd.js.PublishMsg(ctx, nmsg, pubOpts...) @@ -997,60 +1000,65 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid } m := mu.(jetstream.Msg) opts := source.GetOptions() - if dla := opts["DeadLetterAddress"]; dla != "" { - if _, serr := p.ensureStream(ctx, bd, dla); serr != nil { - util.Logger.Warn( - "natsjs: dead-letter of message {0} failed (ensure stream for {1}: {2}); message stays in flight", - uuid, dla, serr.Error()) - return &pb.Error{Message: fmt.Sprintf("dead-letter ensure stream: %s", serr.Error())} - } - // Copied so neither the retry count below nor the publish options - // mutate the original's header map. - dlqHeader := copyHeader(m.Headers()) - var pubOpts []jetstream.PublishOpt - if md, merr := m.Metadata(); merr == nil { - pubOpts = append(pubOpts, - jetstream.WithMsgID(fmt.Sprintf("dlq-%s-%d", md.Stream, md.Sequence.Stream))) - // RabbitMQ stamps a dead-lettered copy broker-side (x-death) with - // how it died; the JetStream copy is a plain publish that would - // otherwise lose that trail. Carry the same retry count the - // consumer saw when it gave the message up, synthesized exactly - // like handleDelivery does for deliveries. - if md.NumDelivered > 1 { - if dlqHeader == nil { - dlqHeader = nats.Header{} - } - dlqHeader.Set(retryCountHeaderName, strconv.FormatUint(md.NumDelivered-1, 10)) + dla, hasDLA := opts["DeadLetterAddress"] + if !hasDLA { + return &pb.Error{Message: "DeadLetterAddress not set"} + } + if dla == "" { + return &pb.Error{Message: "DeadLetterAddress is empty"} + } + if _, serr := p.ensureStream(ctx, bd, dla); serr != nil { + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (ensure stream for {1}: {2}); message stays in flight", + uuid, dla, serr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter ensure stream: %s", serr.Error())} + } + // Copied so neither the retry count below nor the publish options + // mutate the original's header map. + dlqHeader := copyHeader(m.Headers()) + var pubOpts []jetstream.PublishOpt + if md, merr := m.Metadata(); merr == nil { + pubOpts = append(pubOpts, + jetstream.WithMsgID(fmt.Sprintf("dlq-%s-%d", md.Stream, md.Sequence.Stream))) + // RabbitMQ stamps a dead-lettered copy broker-side (x-death) with + // how it died; the JetStream copy is a plain publish that would + // otherwise lose that trail. Carry the same retry count the + // consumer saw when it gave the message up, synthesized exactly + // like handleDelivery does for deliveries. + if md.NumDelivered > 1 { + if dlqHeader == nil { + dlqHeader = nats.Header{} } + dlqHeader.Set(retryCountHeaderName, strconv.FormatUint(md.NumDelivered-1, 10)) } - // RabbitMQ dead-letters a message under its original routing key - // unless the queue overrides it (x-dead-letter-routing-key, which is - // what DeadLetterSubject maps to on the AMQP side). Preserve the - // original key the same way when no override is set, so DLQ consumers - // can bind by routing key and reprocessing tools can see where each - // message was originally headed. - dlSubject := opts["DeadLetterSubject"] - if dlSubject == "" { - dlSubject = routingKeyFromSubject(source.GetAddress().GetName(), m.Subject()) - } - dlqMsg := &nats.Msg{ - Subject: publishSubjectFor(dla, dlSubject), - Data: m.Data(), - Header: dlqHeader, - } - if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { - if errors.Is(perr, jetstream.ErrNoStreamResponse) { - // The DLQ stream was deleted after this connection ensured it. - // Drop the memoized entry so the dead-letter retry that follows - // the returned error (the server nacks, the message redelivers) - // re-creates the stream instead of failing the same way forever. - bd.knownStreams.Delete(streamNameFor(dla)) - } - util.Logger.Warn( - "natsjs: dead-letter of message {0} failed (publish to {1}: {2}); message stays in flight", - uuid, dlqMsg.Subject, perr.Error()) - return &pb.Error{Message: fmt.Sprintf("dead-letter publish: %s", perr.Error())} + } + // RabbitMQ dead-letters a message under its original routing key + // unless the queue overrides it (x-dead-letter-routing-key, which is + // what DeadLetterSubject maps to on the AMQP side). Preserve the + // original key the same way when no override is set, so DLQ consumers + // can bind by routing key and reprocessing tools can see where each + // message was originally headed. + dlSubject := opts["DeadLetterSubject"] + if dlSubject == "" { + dlSubject = routingKeyFromSubject(source.GetAddress().GetName(), m.Subject()) + } + dlqMsg := &nats.Msg{ + Subject: publishSubjectFor(dla, dlSubject), + Data: m.Data(), + Header: dlqHeader, + } + if _, perr := bd.js.PublishMsg(ctx, dlqMsg, pubOpts...); perr != nil { + if errors.Is(perr, jetstream.ErrNoStreamResponse) { + // The DLQ stream was deleted after this connection ensured it. + // Drop the memoized entry so the dead-letter retry that follows + // the returned error (the server nacks, the message redelivers) + // re-creates the stream instead of failing the same way forever. + bd.knownStreams.Delete(streamNameFor(dla)) } + util.Logger.Warn( + "natsjs: dead-letter of message {0} failed (publish to {1}: {2}); message stays in flight", + uuid, dlqMsg.Subject, perr.Error()) + return &pb.Error{Message: fmt.Sprintf("dead-letter publish: %s", perr.Error())} } bd.activeMessages.Delete(uuid) if err := m.Term(); err != nil { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index f08b711..5f07c1a 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -313,6 +313,31 @@ func TestDeadLetterFailureKeepsMessage(t *testing.T) { require.Nil(t, p.Ack(ctx, again.GetUuid())) } +func TestDeadLetterEmptyAddressKeepsMessage(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlempty", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 2) + src := queueSource("events.dlempty.consumer", "events.dlempty", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "an empty DLA must fail instead of terming the original") + assert.Contains(t, perr.GetMessage(), "DeadLetterAddress is empty") + + require.Nil(t, p.Nack(ctx, m.GetUuid()), "a failed dead-letter leaves the message nackable") + again := recv(t, out) + assert.Equal(t, "poison", string(again.GetBody())) + require.Nil(t, p.Ack(ctx, again.GetUuid())) +} + // TestPrefetchMapsToMaxAckPending: a positive PrefetchCount becomes the // consumer's MaxAckPending. A prefetch of 0 is the AMQP "unlimited" // convention — the amqp091 connector honors it by leaving the channel @@ -441,6 +466,35 @@ func TestDeduplication(t *testing.T) { assert.Equal(t, int64(1), stats.GetMessageCount(), "duplicate publish_id within the window is collapsed") } +func TestPublishIDRequiresPublisherName(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.required", Subjects: []string{"e"}}, + Body: []byte("x"), + PublishId: 1, + }) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "PublisherName not set") +} + +func TestPublisherNameWithoutPublishIDDoesNotDedup(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dedup.nameonly", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("a"), PublisherName: "pub"})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("b"), PublisherName: "pub"})) + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.dedup.nameonly.consumer", + Address: &pb.Address{Name: "events.dedup.nameonly"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(2), stats.GetMessageCount(), "publisher_name alone must not collapse messages as pub-0") +} + func TestHeaderFilterDropsNonMatching(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -1225,6 +1279,43 @@ func TestEmptyRoutingKeyDelivers(t *testing.T) { require.Nil(t, p.Ack(ctx, m.GetUuid())) } +func TestDirectAddressBindingIsExact(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.direct.consumer", + Address: &pb.Address{ + Name: "events.direct", + Type: pb.Address_QUEUE, + Subjects: []string{"#"}, + }, + Options: map[string]string{"Offset": "first"}, + } + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: []string{"actual"}}, + Body: []byte("wildcard-leak"), + })) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.direct", Type: pb.Address_QUEUE, Subjects: []string{"#"}}, + Body: []byte("literal-match"), + })) + + m := recv(t, out) + assert.Equal(t, "literal-match", string(m.GetBody()), "direct bindings must not treat # as a topic wildcard") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected direct-address wildcard delivery: %q", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + // TestSourceStatsReportsConsumerBacklog: for a durable source the message // count is the consumer's backlog, not the stream depth — the stream retains // acked messages under its retention limits, so its count never shrinks as From bbdc2611879d253c0260d639cb4a950dbca7617a Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 23:04:39 -0400 Subject: [PATCH 38/77] fix: escape subject tokens injectively instead of sanitizing to '_' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address and routing-key tokens were sanitized by replacing every character NATS forbids in a subject token with '_' (and an empty or bare-'~' address token with '_'), which is not injective: distinct AMQP names merged onto one subject. For addresses that is the worst available failure mode — a.~.b, a..b, a.*.b and a._.b all shared the root a._.b, one subject space, and one stream, so each address received the others' messages and every re-ensure reconfigured the shared stream. Routing keys had the same defect one level down: a binding on a_b also matched published keys 'a b' and 'a*b', and a direct (QUEUE) binding on the literal '#' matched published '_', '*' and '>'. Tokens are now escaped instead of sanitized: a token free of reserved characters ('~', whitespace, '*', '>', '#') passes through unchanged, so every conventional dotted name keeps its historical form and deployed subject layouts do not move. Any other token takes an escaped form marked by a leading '~' (reserved, so no plain token can start with it) in which each reserved character becomes a two-character '~' code; an empty address token becomes '~e'. The escaped form decodes unambiguously, which makes the whole mapping injective, and routingKeyFromSubject now decodes it so a dead-letter republish of a recovered routing key lands on the same tokens instead of escaping them a second time. Escaping CR/LF also removes a latent broken-subject failure for tokens containing them. Only names and keys the previous mapping already merged or broke change their stored subjects; the design doc's persistence-contract paragraph records that, and regression tests pin root injectivity, publish/binding round trips, and end-to-end coexistence of formerly colliding addresses and routing keys. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 71 ++++--- .../provider/connectors/natsjs/helpers.go | 180 ++++++++++++++---- .../connectors/natsjs/helpers_test.go | 126 ++++++++++-- .../provider/connectors/natsjs/natsjs_test.go | 64 +++++++ 4 files changed, 355 insertions(+), 86 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 849b3b4..df0d86a 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -62,30 +62,46 @@ relationship (`events.orders` and `events.orders.filtered`); without the delimiter their capture wildcards overlap, whichever stream is created first wins, and the other address fails every publish and subscribe with `subjects overlap with an existing stream` (err 10065). With it, the token after the -shared prefix differs (`~` vs `filtered`) and sanitization guarantees no -address token ever equals `~`, so any two distinct roots are disjoint. The -delimiter also prevents cross-address message leakage — without it, -publishing routing key `filtered.x` to `events.orders` is indistinguishable -from publishing `x` to `events.orders.filtered`. - -NATS subjects are stricter than AMQP routing keys, so each token is also -sanitized. In binding patterns (`translateWildcards`): runs of consecutive -`#` collapse into one first (they are equivalent in AMQP, and `#.#` must -keep the zero-word match its trailing `#` provides), a non-terminal `#` -becomes `*` (NATS `>` is tail-only), illegal characters (space, tab, `*`, -`>`) inside a literal token become `_`, and empty tokens (from `a..b` or a -trailing dot) are dropped. A pattern whose trailing `#` became `>` also gets -the zero-word variant as a second filter subject (AMQP `#` matches zero or -more words, NATS `>` one or more, so binding `a.#` must match routing key -`a`). On the publish side routing keys are literal — AMQP gives `*`/`#` -meaning only in bindings, and NATS forbids wildcard tokens in published -subjects — so wildcard characters are sanitized to `_` like any other illegal -character. In address names, empty tokens and tokens equal to `~` are -replaced with `_` rather than dropped, so distinct names keep distinct roots. -For `Address_QUEUE` (AMQP direct-exchange parity), binding subjects are exact -routing keys rather than topic patterns: `#` and `*` are literals and are -sanitized the same way a published routing key is sanitized, instead of -matching as wildcards. +shared prefix differs (`~` vs `filtered`) and token escaping (below) +guarantees no address token ever equals `~`, so any two distinct roots are +disjoint. The delimiter also prevents cross-address message leakage — +without it, publishing routing key `filtered.x` to `events.orders` is +indistinguishable from publishing `x` to `events.orders.filtered`. + +NATS subjects are stricter than AMQP routing keys, so each address and +routing-key token is escaped — injectively, so distinct AMQP names never +merge onto one subject. A token free of reserved characters (`~`, +whitespace, `*`, `>`, `#`) passes through unchanged; every conventional +dotted name keeps its readable, historical form. Any other token takes an +escaped form marked by a leading `~` — no plain token can start with `~`, +since `~` is itself reserved — in which each reserved character becomes a +two-character `~` code (`~~` for `~`, `~w` space, `~t` tab, `~a` `*`, `~g` +`>`, `~h` `#`, plus codes for the remaining whitespace characters); an empty +address token (from consecutive dots) becomes `~e`. The escaped form decodes +unambiguously, which is what makes the mapping injective: a lossy +replacement (`_` for every illegal character) would merge distinct addresses +— `a.~.b`, `a..b`, `a.*.b` and `a._.b` — onto one root, and addresses that +share a root share a stream: each receives the other's traffic and each +ensure reconfigures the other's stream. The same escaping keeps distinct +routing keys distinct, so a binding on `a_b` no longer also matches +published keys `a b` or `a*b`. + +In binding patterns (`translateWildcards`): runs of consecutive `#` collapse +into one first (they are equivalent in AMQP, and `#.#` must keep the +zero-word match its trailing `#` provides), a non-terminal `#` becomes `*` +(NATS `>` is tail-only), literal tokens are escaped exactly like published +tokens (so a literal binding matches exactly the published keys it matches +on RabbitMQ), and empty tokens (from `a..b` or a trailing dot) are dropped — +on the publish side too, so both sides agree. A pattern whose trailing `#` +became `>` also gets the zero-word variant as a second filter subject (AMQP +`#` matches zero or more words, NATS `>` one or more, so binding `a.#` must +match routing key `a`). On the publish side routing keys are literal — AMQP +gives `*`/`#` meaning only in bindings, and NATS forbids wildcard tokens in +published subjects — so wildcard characters are escaped like any other +reserved character. For `Address_QUEUE` (AMQP direct-exchange parity), +binding subjects are exact routing keys rather than topic patterns: `#` and +`*` are escaped literals and match only themselves, instead of matching as +wildcards. A redundant binding set — a wildcard binding alongside a specific key it already covers, such as `orders.#` with `orders.created` — is legal in AMQP, @@ -108,7 +124,12 @@ that reads or writes JetStream subjects directly must use the same mapping, including the `~` delimiter. Deployments whose address or source names contain `_`, `/` or `\` predating the escaped name form had those names colliding or rejected outright, so the escaped form changes only names that -were already broken. +were already broken. Likewise, addresses and routing keys containing +reserved subject characters (whitespace, `*`, `>`, `#`, `~`, consecutive +dots) predating token escaping either collided with each other's `_` +spellings or produced invalid subjects, so escaping changes the stored +subjects only where the previous mapping was already wrong; names and keys +made of conventional tokens map exactly as before. AMQP headers-exchange routing has no NATS subject equivalent. It is reproduced proxy-side in `evaluateFilters`: multiple `Filter`s are OR'd (each is a diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 3329172..c2db15b 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -35,12 +35,11 @@ import ( // and the other fails forever with "subjects overlap with an existing stream" // (err 10065). The delimiter makes any two distinct roots disjoint: the token // after "events.orders" is "~" for its own traffic but "filtered" for the -// other address, and "~" is stripped from address tokens by sanitization, so -// no address token can ever equal the delimiter. It also keeps the two -// addresses' messages apart (without it, publishing "filtered.x" to -// "events.orders" would be indistinguishable from publishing "x" to -// "events.orders.filtered"), and gives the empty routing key a concrete -// subject (".~"). +// other address, and token escaping (escapeToken) guarantees no address token +// can ever equal the delimiter. It also keeps the two addresses' messages +// apart (without it, publishing "filtered.x" to "events.orders" would be +// indistinguishable from publishing "x" to "events.orders.filtered"), and +// gives the empty routing key a concrete subject (".~"). // // This is a faithful mapping for the dotted, topic-style keys AMQP topic // exchanges use (e.g. orders.region.us.created.*). It does NOT reproduce AMQP @@ -48,23 +47,118 @@ import ( // (see Subscribe). // addressDelim is the reserved token inserted between the address root and the -// routing-key tokens. Sanitization guarantees no address token equals it. +// routing-key tokens. Token escaping guarantees no address token equals it. const addressDelim = "~" -// addressRoot sanitizes an address name into a valid dotted subject prefix. -// Empty tokens and tokens equal to the reserved delimiter are replaced rather -// than dropped so distinct address names keep distinct roots where possible. +// tokenEscapes encodes, inside an escaped subject token (see escapeToken), +// the characters a token cannot carry literally: the whitespace and wildcard +// characters NATS forbids in a subject token, '#' (so a literal AMQP '#' +// stays distinct from every other token on both the publish and binding +// sides), and '~', which the connector reserves as its address delimiter and +// as the escape marker itself. Every code is two characters starting with +// '~' and no literal character inside an escaped token is ever '~', so an +// escaped token decodes unambiguously left to right — which is what makes +// the encoding injective. +var tokenEscapes = map[rune]string{ + '~': "~~", + ' ': "~w", + '\t': "~t", + '\r': "~r", + '\n': "~n", + '\f': "~f", + '*': "~a", + '>': "~g", + '#': "~h", +} + +// tokenUnescapes inverts tokenEscapes (second code byte -> original byte). +var tokenUnescapes = map[byte]byte{ + '~': '~', 'w': ' ', 't': '\t', 'r': '\r', 'n': '\n', 'f': '\f', + 'a': '*', 'g': '>', 'h': '#', +} + +// tokenEscapeTriggers are the characters that force a token into the escaped +// form; a token free of them passes through literally. +const tokenEscapeTriggers = "~ \t\r\n\f*>#" + +// escapeToken maps one address or routing-key token onto a NATS-legal +// subject token, injectively: distinct tokens never yield the same output. A +// lossy replacement here (the obvious "swap illegal characters for '_'") +// would merge distinct AMQP names — "a b" and "a_b", or an address token +// "~" and "_" — onto one subject, and for addresses that is the worst +// failure mode available: two addresses sharing a subject space share a +// stream, leak each other's messages, and reconfigure each other's topology. +// +// Tokens free of reserved characters pass through unchanged, so every +// conventional dotted name keeps its readable, historical form. Any other +// token takes an escaped form marked by a leading '~' — which no plain token +// can start with, since '~' itself is reserved — followed by the token with +// each reserved character replaced by its two-character code (tokenEscapes). +// The empty token (AMQP allows consecutive dots) becomes the fixed marker +// "~e", which cannot collide with an escaped non-empty token: those always +// contain a second '~', because only tokens containing a reserved character +// are escaped and every reserved character encodes to a '~' pair. +func escapeToken(t string) string { + if t == "" { + return "~e" + } + if !strings.ContainsAny(t, tokenEscapeTriggers) { + return t + } + var b strings.Builder + b.WriteByte('~') + for _, r := range t { + if esc, ok := tokenEscapes[r]; ok { + b.WriteString(esc) + } else { + b.WriteRune(r) + } + } + return b.String() +} + +// unescapeToken inverts escapeToken. Tokens the connector did not escape +// come back unchanged; an escape code the connector never produces is kept +// literally (best effort — such tokens do not occur in subjects the +// connector itself built). Escape codes are pure ASCII and '~' (0x7E) never +// occurs inside a UTF-8 multi-byte sequence, so byte-wise scanning is safe. +func unescapeToken(t string) string { + if !strings.HasPrefix(t, "~") { + return t + } + if t == "~e" { + return "" + } + body := t[1:] + var b strings.Builder + for i := 0; i < len(body); i++ { + c := body[i] + if c != '~' || i+1 == len(body) { + b.WriteByte(c) + continue + } + i++ + if orig, ok := tokenUnescapes[body[i]]; ok { + b.WriteByte(orig) + } else { + b.WriteByte('~') + b.WriteByte(body[i]) + } + } + return b.String() +} + +// addressRoot maps an address name onto a valid dotted subject prefix. +// Tokens are escaped rather than lossily replaced or dropped, so distinct +// address names always keep distinct roots — two addresses may never share a +// subject space (see escapeToken). func addressRoot(addressName string) string { if addressName == "" { return "" } tokens := strings.Split(addressName, ".") for i, t := range tokens { - t = tokenSanitizer.Replace(t) - if t == "" || t == addressDelim { - t = "_" - } - tokens[i] = t + tokens[i] = escapeToken(t) } return strings.Join(tokens, ".") } @@ -82,7 +176,8 @@ func subjectPrefix(addressName string) string { // characters JetStream rejects in stream/consumer names plus '_' itself. // Every code is two characters starting with '_' and no literal character // maps to '_', so an escaped name decodes unambiguously left to right — -// which is what makes the encoding injective. +// which is what makes the encoding injective. (This is the name-level +// counterpart of tokenEscapes: subjects reserve '~', names reserve '_'.) var nameEscapes = map[rune]string{ '.': "_d", '_': "_u", @@ -105,7 +200,7 @@ const nameEscapeTriggers = "_/\\\r\n\f" // '>', '/' or '\'. Derived from the sanitized root so every address that // shares a root shares a stream. // -// The common case — tokens free of '_' and of characters JetStream rejects +// The common case — roots free of '_' and of characters JetStream rejects // in names — keeps the readable historical form, dots swapped for // underscores ("events.orders" -> "arke_events_orders"). That replacement is // only unambiguous while no token contains '_' of its own: "a.b" and "a_b" @@ -114,7 +209,8 @@ const nameEscapeTriggers = "_/\\\r\n\f" // flips the stream's subjects to its own address). Such roots instead take // an escaped form under the disjoint "arke-" prefix, where every '_' starts // a two-character escape code (nameEscapes), so distinct roots always yield -// distinct names across both forms. +// distinct names across both forms. Roots are already injective per address +// (escapeToken), so the composed address -> stream-name mapping is too. func streamNameFor(addressName string) string { root := addressRoot(addressName) if !strings.ContainsAny(root, nameEscapeTriggers) { @@ -135,8 +231,10 @@ func streamNameFor(addressName string) string { // publishSubjectFor maps an address + routing key onto the concrete subject a // message is published to. Routing keys are literal on the publish side — // AMQP only gives '*' / '#' wildcard meaning in bindings — and NATS forbids -// wildcard tokens in published subjects, so they are sanitized like any other -// illegal character. +// wildcard tokens in published subjects, so every token is escaped +// (injectively: distinct routing keys never merge onto one subject, see +// escapeToken). Empty tokens (consecutive or trailing dots) are dropped; +// binding patterns drop them identically, so both sides agree. func publishSubjectFor(addressName, routingKey string) string { prefix := subjectPrefix(addressName) tokens := strings.Split(routingKey, ".") @@ -145,7 +243,7 @@ func publishSubjectFor(addressName, routingKey string) string { if t == "" { continue } - out = append(out, publishSanitizer.Replace(t)) + out = append(out, escapeToken(t)) } if len(out) == 0 { return prefix @@ -153,15 +251,6 @@ func publishSubjectFor(addressName, routingKey string) string { return prefix + "." + strings.Join(out, ".") } -// publishSanitizer is tokenSanitizer plus '#': on the publish side wildcard -// characters carry no meaning and must not appear in the subject. -var publishSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_", "#", "_") - -// tokenSanitizer replaces characters that are illegal inside a NATS subject -// token (space/tab, and the wildcard chars '*'/'>') so a literal AMQP token that -// happens to embed one of them still yields a valid subject. -var tokenSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_") - // translateWildcards converts an AMQP topic routing key into a valid NATS subject // fragment. NATS is stricter than AMQP about subjects, so besides translating the // wildcards it also sanitizes each token. Mapping (per dot-delimited token): @@ -172,8 +261,11 @@ var tokenSanitizer = strings.NewReplacer(" ", "_", "\t", "_", "*", "_", ">", "_" // has no exact equivalent; '*' keeps the subject valid // but narrows zero-or-more to exactly-one. // "" -> dropped (collapses "a..b" and leading/trailing dots, which -// would otherwise be empty — and thus invalid — tokens) -// other -> illegal chars (space/tab/'*'/'>') replaced with '_' +// would otherwise be empty — and thus invalid — tokens; +// publishSubjectFor drops them identically) +// other -> escaped like a published token (escapeToken), so a +// literal binding token matches exactly the +// published keys it matches on RabbitMQ // // Runs of consecutive '#' are collapsed into one before translation — they // are equivalent in AMQP ('#' matches zero or more words) — so "#.#" gets @@ -208,22 +300,30 @@ func translateWildcards(key string) string { out = append(out, "*") } default: - out = append(out, tokenSanitizer.Replace(t)) + out = append(out, escapeToken(t)) } } return strings.Join(out, ".") } // routingKeyFromSubject recovers the routing key a delivered message was -// published with by stripping the address's subject prefix — the inverse of -// publishSubjectFor for subjects the connector itself produced. The bare -// prefix (an empty routing key) and any subject not under the prefix (which -// a consumer's filter subjects make impossible) yield "". +// published with by stripping the address's subject prefix and decoding each +// escaped token — the inverse of publishSubjectFor for subjects the +// connector itself produced (the decode is what keeps a dead-letter +// republish of the recovered key on the same tokens instead of escaping +// them a second time). The bare prefix (an empty routing key) and any +// subject not under the prefix (which a consumer's filter subjects make +// impossible) yield "". func routingKeyFromSubject(addressName, subject string) string { - if rk, ok := strings.CutPrefix(subject, subjectPrefix(addressName)+"."); ok { - return rk + rk, ok := strings.CutPrefix(subject, subjectPrefix(addressName)+".") + if !ok { + return "" } - return "" + tokens := strings.Split(rk, ".") + for i, t := range tokens { + tokens[i] = unescapeToken(t) + } + return strings.Join(tokens, ".") } // streamSubjectsFor returns the subject set a stream captures for the given diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index b7932f4..b8a86c1 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -16,16 +16,16 @@ func TestStreamNameFor(t *testing.T) { assert.Equal(t, "arke_events_audit", streamNameFor("events.audit")) assert.Equal(t, "arke_", streamNameFor("")) - // Names whose tokens contain '_' (directly, or via sanitization of an - // illegal subject char) take the escaped "arke-" form so they cannot - // collide with a dotted sibling: under the plain dots-to-underscores - // replacement "a.b" and "a_b" would both read "arke_a_b" and clobber each - // other's stream. + // Names whose tokens contain '_' take the escaped "arke-" form so they + // cannot collide with a dotted sibling: under the plain + // dots-to-underscores replacement "a.b" and "a_b" would both read + // "arke_a_b" and clobber each other's stream. assert.Equal(t, "arke_a_b", streamNameFor("a.b")) assert.Equal(t, "arke-a_ub", streamNameFor("a_b")) assert.Equal(t, "arke-a_ub_dc", streamNameFor("a_b.c")) - // sanitized chars ('*' -> '_') route through the escaped form too - assert.Equal(t, "arke-a_db_uc", streamNameFor("a.b*c")) + // subject-escaped chars ('*' -> "~a") stay in the plain name form — the + // escaped root contains '~', not '_', and '~' is JetStream-name-legal + assert.Equal(t, "arke_a_~b~ac", streamNameFor("a.b*c")) // characters JetStream rejects in names but allows in subjects are escaped assert.Equal(t, "arke-a_sb", streamNameFor("a/b")) assert.Equal(t, "arke-a_bb", streamNameFor(`a\b`)) @@ -50,13 +50,74 @@ func TestStreamNameFor(t *testing.T) { func TestAddressRoot(t *testing.T) { assert.Equal(t, "events.orders", addressRoot("events.orders")) assert.Equal(t, "", addressRoot("")) - // empty tokens are kept as placeholders, not dropped, so "a..b" and "a.b" - // stay distinct roots - assert.Equal(t, "a._.b", addressRoot("a..b")) - // the reserved delimiter token is sanitized out of address names - assert.Equal(t, "a._.b", addressRoot("a.~.b")) - // illegal subject chars inside tokens are replaced - assert.Equal(t, "a.b_c", addressRoot("a.b*c")) + // empty tokens are kept as escaped placeholders, not dropped, so "a..b" + // and "a.b" stay distinct roots + assert.Equal(t, "a.~e.b", addressRoot("a..b")) + // the reserved delimiter token is escaped, never left bare + assert.Equal(t, "a.~~~.b", addressRoot("a.~.b")) + // illegal subject chars inside tokens are escaped + assert.Equal(t, "a.~b~ac", addressRoot("a.b*c")) + // tokens free of reserved characters — every conventional name — pass + // through unchanged, '_' included + assert.Equal(t, "a._.b-c", addressRoot("a._.b-c")) +} + +// TestAddressRootInjective is the regression test for the root collapse: a +// lossy replacement mapped "a.~.b", "a..b", "a. .b" and "a.*.b" all onto +// "a._.b", so those distinct addresses shared one subject space and one +// stream — cross-address message leakage plus mutual stream reconfiguration. +func TestAddressRootInjective(t *testing.T) { + collisionProne := []string{ + "a._.b", "a.~.b", "a..b", "a. .b", "a.\t.b", "a.*.b", "a.>.b", + "a.#.b", "a.~~.b", "a.~e.b", "a b", "a_b", "a*b", "a\rb", + } + seen := map[string]string{} + for _, addr := range collisionProne { + root := addressRoot(addr) + if prev, ok := seen[root]; ok { + t.Errorf("addressRoot collision: %q and %q both map to %q", prev, addr, root) + } + seen[root] = addr + for _, tok := range strings.Split(root, ".") { + assert.NotEqual(t, addressDelim, tok, "no root token may equal the delimiter (%q)", addr) + assert.NotContainsf(t, tok, " ", "root token must be subject-legal (%q)", addr) + assert.False(t, tok == "" || tok == "*" || tok == ">", + "root token must be a concrete subject token (%q -> %q)", addr, root) + } + } + // pairwise: distinct addresses -> disjoint stream subject spaces and + // distinct stream names (the composed guarantee the delimiter and the + // two escape layers exist to provide) + addrs := collisionProne + for i := range addrs { + for j := i + 1; j < len(addrs); j++ { + for _, p := range streamSubjectsFor(addrs[i]) { + for _, c := range streamSubjectsFor(addrs[j]) { + assert.False(t, subjectsOverlap(p, c), + "addresses %q and %q share subject space (%q vs %q)", addrs[i], addrs[j], p, c) + } + } + assert.NotEqual(t, streamNameFor(addrs[i]), streamNameFor(addrs[j]), + "addresses %q and %q share a stream name", addrs[i], addrs[j]) + } + } +} + +func TestEscapeTokenRoundTrip(t *testing.T) { + for _, tok := range []string{ + "", "plain", "with_underscore", "~", "~~", "~e", "a b", "a\tb", + "a*b", "a>b", "a#b", "a~b", "#", "*", ">", " ", "mixed ~*># end", + "uni-cödé", "uni cödé", + } { + esc := escapeToken(tok) + assert.Equal(t, tok, unescapeToken(esc), "round trip of %q via %q", tok, esc) + assert.NotEqual(t, addressDelim, esc, "escaped token may not equal the delimiter") + assert.NotContainsf(t, esc, " ", "escaped token must be subject-legal (%q)", tok) + } + // tokens the connector never produced pass through the decoder unchanged + // (unknown escape codes are kept literally, best effort) + assert.Equal(t, "plain", unescapeToken("plain")) + assert.Equal(t, "x~q", unescapeToken("~x~q")) } func TestPublishSubjectFor(t *testing.T) { @@ -70,16 +131,27 @@ func TestPublishSubjectFor(t *testing.T) { assert.Equal(t, "~.a.b", publishSubjectFor("", "a.b")) assert.Equal(t, "~", publishSubjectFor("", "")) // wildcard chars are literal in a published routing key and NATS forbids - // them in publish subjects -> sanitized - assert.Equal(t, "events.orders.~.a._.b", publishSubjectFor("events.orders", "a.#.b")) - assert.Equal(t, "events.orders.~.a._", publishSubjectFor("events.orders", "a.*")) + // them in publish subjects -> escaped (injectively: "a.#.b" and "a._.b" + // must not merge onto one subject) + assert.Equal(t, "events.orders.~.a.~~h.b", publishSubjectFor("events.orders", "a.#.b")) + assert.Equal(t, "events.orders.~.a.~~a", publishSubjectFor("events.orders", "a.*")) // empty tokens (double dot / trailing dot) are collapsed, not left invalid assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a..b")) assert.Equal(t, "events.orders.~.a.b", publishSubjectFor("events.orders", "a.b.")) assert.Equal(t, "events.orders.~", publishSubjectFor("events.orders", ".")) - // illegal chars embedded in a literal token are replaced with '_' - assert.Equal(t, "events.orders.~.a_b.c_d", publishSubjectFor("events.orders", "a*b.c>d")) - assert.Equal(t, "events.orders.~.a_b", publishSubjectFor("events.orders", "a b")) + // illegal chars embedded in a literal token are escaped in place + assert.Equal(t, "events.orders.~.~a~ab.~c~gd", publishSubjectFor("events.orders", "a*b.c>d")) + assert.Equal(t, "events.orders.~.~a~wb", publishSubjectFor("events.orders", "a b")) + // distinct routing keys never merge: "a b" vs "a_b", "#" vs "_" vs "*" + distinct := []string{"a b", "a_b", "#", "_", "*", "~", "~e"} + seen := map[string]string{} + for _, rk := range distinct { + s := publishSubjectFor("events.orders", rk) + if prev, ok := seen[s]; ok { + t.Errorf("publish subject collision: %q and %q both map to %q", prev, rk, s) + } + seen[s] = rk + } } func TestRoutingKeyFromSubject(t *testing.T) { @@ -92,6 +164,13 @@ func TestRoutingKeyFromSubject(t *testing.T) { assert.Equal(t, "", routingKeyFromSubject("events.orders", "events.audit.~.x")) // empty address: subjects live directly under the delimiter assert.Equal(t, "a.b", routingKeyFromSubject("", "~.a.b")) + // escaped tokens decode back to the published key, so a dead-letter + // republish of the recovered key lands on the same tokens (round trip: + // rk -> subject -> rk for keys with reserved characters) + for _, rk := range []string{"a b.c", "x.~.y", "#", "a*b", "region.us"} { + subj := publishSubjectFor("events.orders", rk) + assert.Equal(t, rk, routingKeyFromSubject("events.orders", subj), "via %q", subj) + } } func TestStreamSubjectsFor(t *testing.T) { @@ -206,9 +285,14 @@ func TestDirectFilterSubjectsAreExact(t *testing.T) { } assert.Equal(t, - []string{"events.direct.~._"}, + []string{"events.direct.~.~~h"}, filterSubjectsFor(src("#")), "direct bindings treat # as a literal routing key, not a wildcard") + // ...and the literal '#' binding must not also match published "_" or + // "*" (the lossy sanitizer merged all three onto "_") + assert.NotEqual(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "_")) + assert.NotEqual(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "*")) + assert.Equal(t, filterSubjectsFor(src("#"))[0], publishSubjectFor("events.direct", "#")) assert.Equal(t, []string{"events.direct.~.created", "events.direct.~.region.us"}, filterSubjectsFor(src("created", "region.us", "created"))) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 5f07c1a..4e07eef 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -883,6 +883,70 @@ func TestUnderscoreAddressesCoexist(t *testing.T) { } } +// TestEscapedAddressesCoexist: address names "evt.~.b" and "evt._.b" are +// distinct AMQP exchanges, but a lossy token sanitizer mapped both onto the +// root "evt._.b" — one shared subject space and one shared stream, so each +// address received the other's messages and each ensure reconfigured the +// other's stream. Escaped tokens must keep them fully apart. +func TestEscapedAddressesCoexist(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + tilde := &pb.Address{Name: "evt.~.b", Subjects: []string{"k"}} + scored := &pb.Address{Name: "evt._.b", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: tilde, Body: []byte("to-tilde")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: scored, Body: []byte("to-scored")}), + "publishing to evt._.b after evt.~.b must not land on its stream") + + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + for addr, want := range map[string]string{"evt.~.b": "to-tilde", "evt._.b": "to-scored"} { + out := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, queueSource(addr+".consumer", addr, "k"), out) + m := recv(t, out) + assert.Equal(t, want, string(m.GetBody()), "consumer on %q sees its own message", addr) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("unexpected cross-address delivery on %q: %q", addr, extra.GetBody()) + case <-time.After(500 * time.Millisecond): + } + } +} + +// TestEscapedRoutingKeysStayDistinct: routing keys "a b" and "a_b" are +// distinct on a direct exchange, but the lossy sanitizer merged both onto +// "a_b", so a queue bound to "a_b" also received every "a b" message. +func TestEscapedRoutingKeysStayDistinct(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + pub := func(rk, body string) { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "evt.direct", Subjects: []string{rk}}, + Body: []byte(body), + })) + } + pub("a b", "spaced") + pub("a_b", "scored") + + src := queueSource("evt.direct.consumer", "evt.direct", "a_b") + src.Address.Type = pb.Address_QUEUE + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + assert.Equal(t, "scored", string(m.GetBody()), `binding "a_b" must match only routing key "a_b"`) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("binding %q wrongly matched routing key %q message", "a_b", string(extra.GetBody())) + case <-time.After(500 * time.Millisecond): + } +} + // TestUnderscoreSourceNamesStayIndependent: two queues named "grp.a" and // "grp_a" bound to the same address are independent queues — each receives // every message — but a naive name mapping collapses them onto one durable, From d8d9d60715f97f5c7ceefcf8ff32bfeb86fde263 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 23:10:50 -0400 Subject: [PATCH 39/77] fix: end a subscription whose server-side consumer disappeared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live subscription's JetStream consumer can vanish while the client connection survives: deleted administratively, or expired (ephemeral inactivity threshold) during a broker outage — a NATS client reconnects transparently through state changes that would sever an AMQP connection, and RabbitMQ closes a deleted queue's consumer channel so clients notice. Here nothing noticed: the client library treats 'consumer deleted' as terminal and silently stops the consume machinery, and a consumer that expired while disconnected produces only unanswered pulls and missed heartbeats after reconnect. Either way Subscribe kept blocking on its context, so the subscription sat deaf forever — warn-spam at best, silence at worst — until the client process was bounced. Subscribe now ends with a non-fatal error when the consumer is gone, so the client's re-subscribe recreates the consumer (and, after a storage wipe, the stream, via the existing stale-topology heal). Three detectors feed it: the consume context stopping on its own (terminal library errors), authoritative consumer-deleted/not-found errors in the consume error handler, and a bounded single-flight existence probe triggered by the ambiguous errors (missed heartbeats, unanswered pulls) and trusted only when it returns an explicit not-found for the consumer or its stream. Deliberate teardown is distinguished by context/connection state, and Disconnect now marks the connection closed before stopping consume contexts so its own teardown never reads as a lost consumer. Regression tests cover an administratively deleted durable (the terminal path) and a broker restart with a wiped store at the same address (the probe path), each ending the subscription and resuming delivery on re-subscribe. The design doc gains a consumer-loss recovery bullet. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 16 ++- internal/provider/connectors/natsjs/natsjs.go | 99 +++++++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 111 +++++++++++++++++- 3 files changed, 218 insertions(+), 8 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index df0d86a..39d68c5 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -294,7 +294,7 @@ is intended as the primary disk guard. ## Operational resilience -Four mechanisms harden the connector against a cold or busy broker. All are +Five mechanisms harden the connector against a cold or busy broker. All are sized for a clustered (replicated) server, where creating topology also means forming a raft group per stream and per durable consumer: @@ -325,6 +325,20 @@ forming a raft group per stream and per durable consumer: duplicate it), instead of failing every call until the client reconnects. A failed dead-letter publish likewise drops the entry so the retry that follows re-creates the DLQ stream. +- **Consumer-loss recovery.** The same class of event can take a live + subscription's server-side consumer: deleted administratively, or expired + (ephemeral inactivity threshold) during an outage the client connection + survives. The client library either stops delivering silently — it treats + "consumer deleted" as terminal — or keeps pulling a consumer that no + longer answers, and RabbitMQ has no analogous state: a deleted queue + closes its consumers' channel. When the consume machinery stops on its + own, when an authoritative consumer-gone error arrives, or when a bounded + single-flight probe (triggered by consume errors such as missed + heartbeats or unanswered pulls, and trusted only for an explicit "not + found" answer) confirms the consumer is gone, `Subscribe` ends with a + non-fatal error. The client's re-subscribe then recreates the consumer — + and, after a storage wipe, the stream. A recreated durable starts from + its configured `Offset`, like a re-declared queue starts empty. ## Configuration diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index c5d97ea..341406a 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -411,6 +411,10 @@ func (p *natsjsProvider) Disconnect(ctx context.Context) { if bd == nil { return } + // Mark the connection closed before stopping its consume contexts: + // stopping one wakes its Subscribe (Closed() fires), which reads the + // state to tell deliberate teardown from a consumer lost on the server. + bd.state.Store(provider.CLOSED) for _, name := range bd.consumeContexts.GetList() { if cc, ok := bd.consumeContexts.Get(name); ok { cc.(jetstream.ConsumeContext).Stop() @@ -419,7 +423,6 @@ func (p *natsjsProvider) Disconnect(ctx context.Context) { if bd.nc != nil { _ = bd.nc.Drain() } - bd.state.Store(provider.CLOSED) p.connections.Delete(clientID) util.Logger.Debugf("natsjs: client %s disconnected", clientID) } @@ -688,6 +691,46 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c return nil } + // consumerLost carries the first authoritative sign that this + // subscription's server-side consumer no longer exists (deleted + // administratively, or expired after an outage longer than the ephemeral + // inactivity threshold). The final wait below turns it into a + // subscription-ending error so the client's re-subscribe recreates the + // consumer — without it the subscription would sit deaf forever, warning + // (or, after a terminal consume error, silently) while delivering + // nothing. RabbitMQ parity: a deleted queue closes its consumers' + // channel, and the re-subscribe re-declares it. + consumerName := cons.CachedInfo().Name + consumerLost := make(chan error, 1) + noteLost := func(err error) { + select { + case consumerLost <- err: + default: + } + } + // probeConsumer asks the server whether the consumer still exists. A + // consumer that vanished while no pull was pending (expiry during a long + // outage) never produces an authoritative error on the consume path — + // re-issued pulls just find no responder and heartbeats go missing, + // indefinitely — so those symptoms trigger this bounded, single-flight + // lookup instead. Only an explicit "not found" answer counts (for the + // consumer, or for the whole stream — a wiped store takes both): a + // timeout or network failure proves nothing about the consumer. + var probing atomic.Bool + probeConsumer := func() { + if bd.state.Load() != provider.CONNECTED || !probing.CompareAndSwap(false, true) { + return + } + go func() { + defer probing.Store(false) + pctx, pcancel := context.WithTimeout(context.WithoutCancel(ctx), ephemeralDeleteTimeout) + defer pcancel() + if _, perr := stream.Consumer(pctx, consumerName); errors.Is(perr, jetstream.ErrConsumerNotFound) || + errors.Is(perr, jetstream.ErrStreamNotFound) { + noteLost(perr) + } + }() + } consumeOpts := []jetstream.PullConsumeOpt{ // A short idle heartbeat plus an error handler keep a stalled delivery // path from failing silently: if the server stops serving this @@ -700,6 +743,14 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c jetstream.PullHeartbeat(defaultConsumeHeartbeat), jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { util.Logger.Warn("natsjs: consume error on source {0}: {1}", source.GetName(), err.Error()) + // The deleted/not-found errors are authoritative; anything else + // (missed heartbeats, pulls with no responder) is only a symptom + // that warrants checking. + if errors.Is(err, jetstream.ErrConsumerDeleted) || errors.Is(err, jetstream.ErrConsumerNotFound) { + noteLost(err) + return + } + probeConsumer() }), } // Pull requests must carry the priority group exactly when the consumer @@ -722,7 +773,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c }, consumeOpts...) if cerr != nil { if consCfg.Durable == "" { - deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + deleteEphemeralConsumer(ctx, stream, consumerName) } return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} } @@ -741,14 +792,50 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } _ = bd.nc.FlushTimeout(ephemeralDeleteTimeout) bd.consumeContexts.Delete(subKey) - releaseInFlight(bd, streamName, cons.CachedInfo().Name, consCfg.Durable != "") + releaseInFlight(bd, streamName, consumerName, consCfg.Durable != "") if consCfg.Durable == "" { - deleteEphemeralConsumer(ctx, stream, cons.CachedInfo().Name) + deleteEphemeralConsumer(ctx, stream, consumerName) } }() - <-ctx.Done() - return nil + select { + case <-ctx.Done(): + return nil + case lerr := <-consumerLost: + return consumerLostError(source, consumerName, lerr) + case <-cc.Closed(): + // The consume machinery stopped on its own: the library treats a + // handful of consume errors as terminal ("consumer deleted" when the + // server removes the consumer under a pending pull) and silently + // stops the ConsumeContext. Blocking on ctx here regardless would + // leave a subscription that never delivers again. Closed() also + // fires for this connection's own teardown (Disconnect stops every + // consume context, Drain closes the subscriptions), which is not an + // error — the context or connection state says which case this is. + if ctx.Err() != nil || bd.state.Load() == provider.CLOSED { + return nil + } + select { + case lerr := <-consumerLost: + return consumerLostError(source, consumerName, lerr) + default: + } + return &pb.Error{Message: fmt.Sprintf( + "source %q: consuming stopped unexpectedly (consumer %s); re-subscribe to recreate the consumer", + source.GetName(), consumerName)} + } +} + +// consumerLostError is the non-fatal error Subscribe ends with when the +// server-side consumer disappeared: non-fatal so the client's re-subscribe +// path runs Subscribe again, which recreates the consumer (and, after a +// storage wipe, the stream). Matches the amqp091 connector, which ends a +// subscription with a non-fatal error when the broker closes the channel of +// a deleted queue. +func consumerLostError(source *pb.Source, consumerName string, err error) *pb.Error { + return &pb.Error{Message: fmt.Sprintf( + "source %q: server-side consumer %s no longer exists (%s); re-subscribe to recreate it", + source.GetName(), consumerName, err.Error())} } // releaseInFlight drops a just-ended subscription's unresolved deliveries from diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 4e07eef..6f361ee 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -30,10 +30,18 @@ func init() { // runJetStreamServer starts an in-process NATS server with JetStream enabled, // backed by a temp store dir that is cleaned up with the test. func runJetStreamServer(t *testing.T) *natsserver.Server { + t.Helper() + return runJetStreamServerAt(t, -1) // choose a free port +} + +// runJetStreamServerAt is runJetStreamServer on a caller-chosen port, so a +// test can restart "the broker" at the address a client is connected to — +// with a fresh store dir, i.e. a broker that lost its JetStream state. +func runJetStreamServerAt(t *testing.T, port int) *natsserver.Server { t.Helper() opts := &natsserver.Options{ Host: "127.0.0.1", - Port: -1, // choose a free port + Port: port, JetStream: true, StoreDir: t.TempDir(), NoLog: true, @@ -425,6 +433,107 @@ func TestDeadLetterRecoversDeletedDLQStream(t *testing.T) { "the dead-letter retry must re-create the DLQ stream") } +// TestSubscribeEndsWhenConsumerDeleted: deleting a subscription's server-side +// consumer out from under it (an operator cleanup, a misdirected tool) used +// to leave the subscription permanently deaf — the client library treats the +// resulting consume error as terminal and silently stops delivering, while +// Subscribe kept blocking on its context. Subscribe must instead end with a +// non-fatal error so the client's re-subscribe recreates the consumer. +func TestSubscribeEndsWhenConsumerDeleted(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.zap", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + src := queueSource("events.zap.consumer", "events.zap", "k") + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, out) }() + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.zap")) + require.NoError(t, serr) + require.NoError(t, stream.DeleteConsumer(ctx, durableName(src))) + + select { + case perr := <-done: + require.NotNil(t, perr, "Subscribe must end with an error, not block forever") + assert.False(t, perr.GetIsFatal(), "consumer loss must be retriable (non-fatal)") + case <-time.After(30 * time.Second): + t.Fatal("Subscribe kept blocking after its consumer was deleted") + } + + // The re-subscribe — the client's reaction to the ended stream — + // recreates the durable and delivery resumes. (Offset "first" replays the + // retained backlog into the fresh durable, so drain until the new + // message arrives.) + out2 := make(chan *pb.Message, 4) + go p.Subscribe(subCtx, src, out2) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m2")})) + for { + m2 := recv(t, out2) + require.Nil(t, p.Ack(ctx, m2.GetUuid())) + if string(m2.GetBody()) == "m2" { + break + } + } +} + +// TestSubscribeRecoversAfterServerStateWipe: a broker restart that loses its +// JetStream state (storage wipe, factory reset) does not sever a NATS client +// the way it severs an AMQP client — the connection reconnects transparently +// and the subscription keeps pulling a consumer that no longer exists. No +// terminal error ever arrives on the consume path: pulls find no responder +// and heartbeats go missing, indefinitely. The existence probe must notice +// the consumer is gone and end the subscription (non-fatal), so the client's +// re-subscribe rebuilds stream and consumer. +func TestSubscribeRecoversAfterServerStateWipe(t *testing.T) { + s := runJetStreamServer(t) + port := s.Addr().(*net.TCPAddr).Port + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.wipe3", Subjects: []string{"k"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + src := queueSource("events.wipe3.consumer", "events.wipe3", "k") + out := make(chan *pb.Message, 2) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, out) }() + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // Restart the broker at the same address with an empty store. The client + // reconnects on its own; stream and consumer are gone server-side. + s.Shutdown() + s.WaitForShutdown() + runJetStreamServerAt(t, port) + + select { + case perr := <-done: + require.NotNil(t, perr, "Subscribe must end with an error, not starve silently") + assert.False(t, perr.GetIsFatal(), "state loss must be retriable (non-fatal)") + case <-time.After(60 * time.Second): + t.Fatal("Subscribe kept blocking after the broker lost its JetStream state") + } + + // The re-subscribe rebuilds the whole topology on the wiped broker and + // delivery resumes. + out2 := make(chan *pb.Message, 2) + go p.Subscribe(subCtx, src, out2) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m2")})) + m2 := recv(t, out2) + assert.Equal(t, "m2", string(m2.GetBody())) + require.Nil(t, p.Ack(ctx, m2.GetUuid())) +} + func TestPrefetchMapsToMaxAckPending(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) From ca43e223fe433602131fdcebe039da7c728ad8ed Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 12 Jul 2026 23:13:03 -0400 Subject: [PATCH 40/77] fix: scope stream-ensure coalescing to the connection's credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registry that collapses concurrent ensureStream calls keyed its entries by host, port, and stream name only, while the collapsed CreateOrUpdateStream ran on whichever connection arrived first. Two connections to the same endpoint authenticating as different users — on a multi-account server, different accounts with disjoint JetStream state and permissions — could therefore coalesce onto one call and share an outcome that belongs to only one of them: a stream created in the wrong account, or one account's permission failure handed to a caller that was itself allowed. Failures are not cached and the publish path self-heals afterwards, but the initial ensure could still fail spuriously. The key now includes the connection's username (streamEnsureKey), so same-account stampedes still collapse — the point of the registry — while distinct accounts never share. The stream name stays last in the key and can never contain '/', so an embedded '/' in a username cannot alias two targets. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 7 +++-- internal/provider/connectors/natsjs/natsjs.go | 27 +++++++++++++---- .../provider/connectors/natsjs/natsjs_test.go | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 39d68c5..10eaa18 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -307,8 +307,11 @@ forming a raft group per stream and per durable consumer: are collapsed provider-wide: when many clients (re)connect at once — a proxy restart, a mass reconnect after a broker outage — exactly one `CreateOrUpdateStream` per stream is in flight at a time, and concurrent - callers share its result. Failures are not cached, and success is still - memoized per connection, so a fresh connection re-asserts its topology. + callers share its result. Collapsing is scoped to the broker endpoint plus + the connection's credential identity, so distinct accounts on one server + (disjoint JetStream state and permissions) never share an outcome. + Failures are not cached, and success is still memoized per connection, so + a fresh connection re-asserts its topology. - **Consumer liveness.** Consumers run with a 5s idle heartbeat and a consume error handler. If the server stops serving a consumer's pulls (a broker restart, or a just-created consumer whose raft group is not yet serving), diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 341406a..beb5389 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -131,10 +131,11 @@ type natsjsProvider struct { // e.g. after a broker or proxy restart — every connection would otherwise // issue its own CreateOrUpdateStream for the same shared streams, piling // redundant load on the JetStream metadata leader exactly when it is busiest. -// Entries are keyed by broker endpoint + stream name so distinct brokers never -// share results. Only in-flight calls are tracked here; success is memoized -// per connection (knownStreams), preserving the property that a fresh -// connection re-asserts its topology. +// Entries are keyed by broker endpoint + credential identity + stream name +// (streamEnsureKey) so distinct brokers — or distinct accounts on one broker +// — never share results. Only in-flight calls are tracked here; success is +// memoized per connection (knownStreams), preserving the property that a +// fresh connection re-asserts its topology. type streamRegistry struct { mu sync.Mutex inflight map[string]*inflightEnsure @@ -149,6 +150,21 @@ func newStreamRegistry() *streamRegistry { return &streamRegistry{inflight: make(map[string]*inflightEnsure)} } +// streamEnsureKey identifies one stream-creation target for the registry: +// broker endpoint, credential identity, stream name. The credential part is +// what keeps two connections that share an endpoint but authenticate as +// different users — on a multi-account server, different accounts with +// disjoint JetStream state and permissions — from coalescing onto one +// CreateOrUpdateStream, which would run under whichever account got there +// first and hand its outcome (success in the wrong account, or that +// account's permission failure) to the other. The stream name goes last and +// can never contain '/' (see streamNameFor), so an embedded '/' in a +// username cannot make two distinct targets read alike. +func streamEnsureKey(cfg *pb.ConnectionConfiguration, streamName string) string { + return fmt.Sprintf("%s:%d/%s/%s", + cfg.GetHost(), cfg.GetPort(), cfg.GetCredentials().GetUsername(), streamName) +} + // ensure runs create once per key at a time; callers that arrive while a call // for the same key is in flight wait for it and share its result. Failures // are not cached — the next caller retries. @@ -439,8 +455,7 @@ func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails if _, ok := bd.knownStreams.Get(streamName); ok { return streamName, nil } - key := fmt.Sprintf("%s:%d/%s", bd.connectionConfig.GetHost(), bd.connectionConfig.GetPort(), streamName) - err := p.streams.ensure(ctx, key, func() error { + err := p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), func() error { cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jsAPITimeout()) defer cancel() _, err := bd.js.CreateOrUpdateStream(cctx, jetstream.StreamConfig{ diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 6f361ee..d8e4acb 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -815,6 +815,35 @@ func TestStreamRegistryFollowerHonorsContext(t *testing.T) { close(release) } +// TestStreamEnsureKeyScopedToCredentials: the registry must never coalesce +// two connections that share a broker endpoint but authenticate as different +// users — on a multi-account server those are different accounts with +// disjoint JetStream state and permissions, so sharing one +// CreateOrUpdateStream call would hand one account the other's outcome +// (success in the wrong account, or a permission failure that is not its +// own). +func TestStreamEnsureKeyScopedToCredentials(t *testing.T) { + cfg := func(user string) *pb.ConnectionConfiguration { + c := &pb.ConnectionConfiguration{Host: "nats", Port: 4222} + if user != "" { + c.Credentials = &pb.Credentials{Username: user, Password: "s"} + } + return c + } + assert.Equal(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("a"), "arke_x"), + "same endpoint, account, and stream must coalesce") + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("b"), "arke_x"), + "different accounts must not share an ensure result") + assert.NotEqual(t, streamEnsureKey(cfg(""), "arke_x"), streamEnsureKey(cfg("a"), "arke_x"), + "anonymous and authenticated connections must not share") + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(cfg("a"), "arke_y"), + "different streams must not share") + other := cfg("a") + other.Port = 4223 + assert.NotEqual(t, streamEnsureKey(cfg("a"), "arke_x"), streamEnsureKey(other, "arke_x"), + "different endpoints must not share") +} + func TestDeliverPolicyFor(t *testing.T) { mk := func(off string) *pb.Source { return &pb.Source{Options: map[string]string{"Offset": off}} } check := func(off string, wantPol jetstream.DeliverPolicy, wantSeq uint64) { From ebc12812748efb9c73268c283b3541665684158f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:10:43 -0400 Subject: [PATCH 41/77] fix: release only the closing subscription's in-flight messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several subscriptions can share one server-side JetStream consumer: two subscribers of a durable queue compete for its messages, and single-active standbys attach to the same pinned consumer. Teardown identified a subscription's delivered-but-unresolved messages by the (stream, consumer) pair the delivery carries, which is shared by all of them — so ending one subscription nak'd and dropped every in-flight message on the consumer, including ones a sibling subscription was still holding. The sibling's later ack then failed with "no message with uuid ..." and the nak redelivered the message as a duplicate, on the ordinary competing-consumer shutdown path. Record the owning subscription on each active message (inflightMsg carries the delivery plus a per-subscription subKey) and release only the deliveries the closing subscription owns. Concurrent resolution during teardown is unchanged: whichever side deletes the uuid first wins and the loser's ack/nak is ignored (at-least-once either way). Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 73 ++++++++++++------- .../provider/connectors/natsjs/natsjs_test.go | 44 +++++++++++ 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index beb5389..9e9d465 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -97,8 +97,10 @@ type natsBrokerDetails struct { clientIdentifier string connectionConfig *pb.ConnectionConfiguration - // activeMessages maps an Arke message UUID -> the in-flight jetstream.Msg - // so a later Ack/Nack/Retry/DeadLetter RPC can resolve it. + // activeMessages maps an Arke message UUID -> an inflightMsg (the in-flight + // jetstream.Msg plus the subscription that delivered it) so a later + // Ack/Nack/Retry/DeadLetter RPC can resolve it and teardown can release + // only the closing subscription's own deliveries. activeMessages *util.ConcurrentMap // knownStreams memoizes streams we have already ensured. knownStreams *util.ConcurrentMap @@ -120,6 +122,18 @@ type natsBrokerDetails struct { produced int64 } +// inflightMsg is a delivered-but-unresolved message together with the +// subscription (subKey) that delivered it. Recording the owner lets teardown +// release only the closing subscription's deliveries even when several +// subscriptions share one server-side consumer (competing consumers on a +// durable, single-active standbys) — otherwise closing one subscription would +// nak and drop a sibling's in-flight messages, failing the sibling's later +// ack and duplicating the message. +type inflightMsg struct { + msg jetstream.Msg + subKey string +} + type natsjsProvider struct { connections *util.ConcurrentMap // streams collapses concurrent stream-creation calls across connections. @@ -783,8 +797,13 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c "(broker predates them, or an existing durable could not be updated); consumers will compete", source.GetName()) } + // subKey tags this subscription: it keys its consume context and stamps the + // messages it delivers (inflightMsg), so teardown releases only its own + // deliveries even when it shares a server-side consumer with siblings. + // Generated before Consume so the delivery callback can capture it. + subKey := fmt.Sprintf("%s#%d", source.GetName(), bd.subscriptionSeq.Add(1)) cc, cerr := cons.Consume(func(m jetstream.Msg) { - p.handleDelivery(ctx, bd, source, m, out) + p.handleDelivery(ctx, bd, source, subKey, m, out) }, consumeOpts...) if cerr != nil { if consCfg.Durable == "" { @@ -792,7 +811,6 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } return &pb.Error{Message: fmt.Sprintf("consume: %s", cerr.Error()), IsFatal: true} } - subKey := fmt.Sprintf("%s#%d", source.GetName(), bd.subscriptionSeq.Add(1)) bd.consumeContexts.Add(subKey, cc) defer func() { cc.Stop() @@ -807,7 +825,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } _ = bd.nc.FlushTimeout(ephemeralDeleteTimeout) bd.consumeContexts.Delete(subKey) - releaseInFlight(bd, streamName, consumerName, consCfg.Durable != "") + releaseInFlight(bd, subKey, consCfg.Durable != "") if consCfg.Durable == "" { deleteEphemeralConsumer(ctx, stream, consumerName) } @@ -854,33 +872,38 @@ func consumerLostError(source *pb.Source, consumerName string, err error) *pb.Er } // releaseInFlight drops a just-ended subscription's unresolved deliveries from -// activeMessages. Acks travel on the consume stream that closed, so nothing -// can resolve these uuids anymore; left alone they would sit in the map for -// the life of the connection (a leak that also inflates the active-message -// stat). Durable consumers additionally get a best-effort Nak so the messages -// redeliver promptly — RabbitMQ requeues a closed channel's unacked deliveries -// immediately, whereas an untouched JetStream delivery waits out the full -// AckWait. Ephemeral consumers skip the Nak: the consumer is deleted on this -// same teardown path, which discards its delivery state anyway. A message the -// client resolves concurrently with teardown is deleted by whichever side gets -// there first; the loser's ack/nak fails and is ignored (at-least-once either -// way). -func releaseInFlight(bd *natsBrokerDetails, streamName, consumerName string, nak bool) { +// activeMessages, identified by the owning subKey. Acks travel on the consume +// stream that closed, so nothing can resolve these uuids anymore; left alone +// they would sit in the map for the life of the connection (a leak that also +// inflates the active-message stat). Durable consumers additionally get a +// best-effort Nak so the messages redeliver promptly — RabbitMQ requeues a +// closed channel's unacked deliveries immediately, whereas an untouched +// JetStream delivery waits out the full AckWait. Ephemeral consumers skip the +// Nak: the consumer is deleted on this same teardown path, which discards its +// delivery state anyway. +// +// Ownership is by subKey, not by the (stream, consumer) pair the delivery +// carries: several subscriptions can share one server-side consumer (competing +// consumers on a durable, single-active standbys), so filtering on the +// consumer would release a sibling's still-in-flight messages — failing the +// sibling's later ack and duplicating the message. A message the owner +// resolves concurrently with teardown is deleted by whichever side gets there +// first; the loser's ack/nak fails and is ignored (at-least-once either way). +func releaseInFlight(bd *natsBrokerDetails, subKey string, nak bool) { for _, uuid := range bd.activeMessages.GetList() { mu, ok := bd.activeMessages.Get(uuid) if !ok { continue } - m := mu.(jetstream.Msg) - md, err := m.Metadata() - if err != nil || md.Stream != streamName || md.Consumer != consumerName { + im := mu.(inflightMsg) + if im.subKey != subKey { continue } bd.activeMessages.Delete(uuid) if !nak { continue } - if err := m.Nak(); err != nil { + if err := im.msg.Nak(); err != nil { util.Logger.Debugf("natsjs: nak of in-flight message %s at subscription end: %s", uuid, err.Error()) } } @@ -985,7 +1008,7 @@ func isStartPositionConflict(err error) bool { return false } -func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, m jetstream.Msg, out chan<- *pb.Message) { +func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, subKey string, m jetstream.Msg, out chan<- *pb.Message) { defer util.RecoverPanic() headers := natsToPbHeader(m.Headers()) @@ -1008,7 +1031,7 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai } uuid := util.GenUUID() - bd.activeMessages.Add(uuid, m) + bd.activeMessages.Add(uuid, inflightMsg{msg: m, subKey: subKey}) pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: m.Data()} select { @@ -1034,7 +1057,7 @@ func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstrea return nil, bd, &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} } bd.activeMessages.Delete(uuid) - return mu.(jetstream.Msg), bd, nil + return mu.(inflightMsg).msg, bd, nil } func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { @@ -1100,7 +1123,7 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid if !ok { return &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} } - m := mu.(jetstream.Msg) + m := mu.(inflightMsg).msg opts := source.GetOptions() dla, hasDLA := opts["DeadLetterAddress"] if !hasDLA { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index d8e4acb..fe4f2e0 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1677,6 +1677,50 @@ func TestTeardownReleasesInFlightMessages(t *testing.T) { } } +// TestTeardownReleasesOnlyItsOwnInFlight: two subscriptions with the same +// (durable) source name attach to one server-side consumer and compete, like +// two service instances on a shared queue. When the idle sibling ends, teardown +// must release only its own deliveries — not nak and drop the message the other +// subscription is still holding, which would fail that subscription's ack and +// redeliver the message as a duplicate. +func TestTeardownReleasesOnlyItsOwnInFlight(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.siblings", Subjects: []string{"e"}} + src := queueSource("events.siblings.consumer", "events.siblings", "e") + + // Subscription A takes one message and holds it unacked. + outA := make(chan *pb.Message, 1) + ctxA, cancelA := context.WithCancel(ctx) + defer cancelA() + errA := make(chan *pb.Error, 1) + go func() { errA <- p.Subscribe(ctxA, src, outA) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("held")})) + held := recv(t, outA) // delivered to A, deliberately left unacked + + // Subscription B joins the same durable as a competing sibling, then ends + // without ever receiving anything of its own. + outB := make(chan *pb.Message, 1) + ctxB, cancelB := context.WithCancel(ctx) + errB := make(chan *pb.Error, 1) + go func() { errB <- p.Subscribe(ctxB, src, outB) }() + time.Sleep(400 * time.Millisecond) // let B's pull reach the server + cancelB() + require.Nil(t, <-errB) + + // A's held message must still resolve — the sibling teardown owned none of it. + require.Nil(t, p.Ack(ctx, held.GetUuid()), + "a sibling subscription's teardown must not release this subscription's in-flight message") + + // And it must not have been redelivered (which a wrongful nak would cause). + select { + case dup := <-outA: + t.Fatalf("message redelivered after a sibling teardown: %q", dup.GetBody()) + case <-time.After(300 * time.Millisecond): + } +} + // TestRedundantBindingsCoexist: an AMQP binding set may be redundant — a // wildcard binding alongside a specific key it already covers — and RabbitMQ // routes each message to the queue once no matter how many bindings match. From 87de99474ce356179fb874e883c81172fb034944 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:17:15 -0400 Subject: [PATCH 42/77] fix: reject conflicting header filters on a shared durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Header-filter (headers-exchange) matching has no JetStream equivalent, so it is evaluated proxy-side per delivery against the filter captured by that particular Subscribe call. When two subscribers share one durable consumer — competing consumers on a queue, or a source whose definition is being changed from filter A to filter B while both are live — JetStream assigns each message to exactly one of them. A message meant for the B subscriber that lands on the A subscriber is tested against A's filter, does not match, and is acked and dropped: it never reaches B. RabbitMQ header bindings are queue-wide and cannot diverge this way, so this is silent message loss with no analogue on the broker being emulated. Reject the divergence instead of losing messages: a durable records the header-filter fingerprint of its live subscribers (reference counted, so the last subscriber to leave forgets it and a later re-subscribe may declare a new filter), and a subscriber whose header filter disagrees is refused with a fatal error naming the conflict. Competing consumers that share a header filter — the common case, including single-active groups, which carry none — are unaffected. Only the proxy-side header filters are fingerprinted; subject filters are the server's and update the shared consumer in place. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 92 +++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 64 +++++++++++++ 2 files changed, 156 insertions(+) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 9e9d465..8d6a8bb 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "sort" "strconv" "strings" "sync" @@ -117,11 +118,31 @@ type natsBrokerDetails struct { // rates derives SourceStats publish/deliver rates from JetStream counters. rates *rateTracker + // durableFilters guards competing consumers on one durable against applying + // different proxy-side header filters. Each consumer evaluates only its own + // Subscribe's filters (evaluateFilters in handleDelivery), so if two + // subscribers of one durable disagree, a message JetStream hands to the + // "wrong" one is filtered out and lost to the subscriber that wanted it. + // Keyed by durable name; the entry records the header-filter fingerprint the + // live subscribers share and how many hold it, so a conflicting declaration + // is rejected instead of silently dropping messages. Subject filters are + // excluded — they are the server's and update in place. + durableFilters map[string]*durableFilterUse + durableFilterMu sync.Mutex + state atomic.Uint32 consumed int64 produced int64 } +// durableFilterUse records the shared header-filter fingerprint of a durable's +// live subscribers and a reference count so the entry lives exactly as long as +// at least one subscriber holds it (see natsBrokerDetails.durableFilters). +type durableFilterUse struct { + fingerprint string + refs int +} + // inflightMsg is a delivered-but-unresolved message together with the // subscription (subKey) that delivered it. Recording the owner lets teardown // release only the closing subscription's deliveries even when several @@ -362,6 +383,7 @@ func (p *natsjsProvider) Connect(ctx context.Context, cfg *pb.ConnectionConfigur knownStreams: util.NewConcurrentMap(), consumeContexts: util.NewConcurrentMap(), rates: newRateTracker(), + durableFilters: make(map[string]*durableFilterUse), } bd.state.Store(provider.CONNECTING) @@ -680,6 +702,19 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c consCfg.InactiveThreshold = defaultInactiveThreshold } + // Competing consumers on one durable each apply their own proxy-side header + // filter, so reject a second live subscriber whose header filter differs + // rather than silently dropping the messages JetStream routes to the + // "wrong" one (see natsBrokerDetails.durableFilters). Claimed before the + // consumer is touched and held for the life of this Subscribe; the defer + // covers every return path below. + if consCfg.Durable != "" { + if err := bd.claimDurableFilter(consCfg.Durable, headerFilterFingerprint(source.GetFilters())); err != nil { + return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), err.Error()), IsFatal: true} + } + defer bd.releaseDurableFilter(consCfg.Durable) + } + cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) if cerr != nil && consCfg.Durable != "" && isStartPositionConflict(cerr) { // A durable's start position (DeliverPolicy/OptStartSeq) is fixed at @@ -909,6 +944,63 @@ func releaseInFlight(bd *natsBrokerDetails, subKey string, nak bool) { } } +// claimDurableFilter registers this subscription's header-filter fingerprint +// against a durable. The first subscriber sets the fingerprint; later +// subscribers must match it (they share the server-side consumer, so their +// proxy-side header filters must agree or one drops the other's messages — +// see natsBrokerDetails.durableFilters). A mismatch is refused; a match takes +// a reference. Every successful claim must be paired with releaseDurableFilter. +func (bd *natsBrokerDetails) claimDurableFilter(durable, fingerprint string) error { + bd.durableFilterMu.Lock() + defer bd.durableFilterMu.Unlock() + if use, ok := bd.durableFilters[durable]; ok { + if use.fingerprint != fingerprint { + return fmt.Errorf( + "durable consumer %q already has a live subscriber with a different header filter; "+ + "competing consumers on one durable must share their header filters", durable) + } + use.refs++ + return nil + } + bd.durableFilters[durable] = &durableFilterUse{fingerprint: fingerprint, refs: 1} + return nil +} + +// releaseDurableFilter drops one reference taken by claimDurableFilter, +// forgetting the durable's fingerprint once the last live subscriber leaves so +// a later re-subscribe may legitimately declare a new filter. +func (bd *natsBrokerDetails) releaseDurableFilter(durable string) { + bd.durableFilterMu.Lock() + defer bd.durableFilterMu.Unlock() + if use, ok := bd.durableFilters[durable]; ok { + use.refs-- + if use.refs <= 0 { + delete(bd.durableFilters, durable) + } + } +} + +// headerFilterFingerprint canonically (order-independently) serializes a +// source's proxy-side header filters so two subscriptions sharing a durable can +// be compared. Only the header filters (evaluateFilters) are fingerprinted; +// subject filters are the server's and update the shared consumer in place. +func headerFilterFingerprint(filters []*pb.Filter) string { + if len(filters) == 0 { + return "" + } + parts := make([]string, 0, len(filters)) + for _, f := range filters { + matches := make([]string, 0, len(f.GetMatches())) + for _, m := range f.GetMatches() { + matches = append(matches, fmt.Sprintf("%q=%q", m.GetName(), m.GetValue())) + } + sort.Strings(matches) + parts = append(parts, fmt.Sprintf("t%d[%s]", f.GetType(), strings.Join(matches, ","))) + } + sort.Strings(parts) + return strings.Join(parts, ";") +} + // deleteEphemeralConsumer removes an ephemeral consumer as soon as its // subscription ends. Ephemeral consumers do expire on their own after // InactiveThreshold, but until then each one still counts against the server's diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index fe4f2e0..c03aec3 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -635,6 +635,70 @@ func TestHeaderFilterDropsNonMatching(t *testing.T) { } } +// TestConflictingHeaderFiltersOnDurableRejected: header filters are evaluated +// proxy-side per consumer, so two competing subscribers of one durable that +// disagree would each drop the other's messages (JetStream hands a message to +// one of them, and if it does not match that one's filter it is acked and +// lost). RabbitMQ header bindings are queue-wide and cannot diverge like this, +// so reject the conflicting declaration instead of silently losing messages. +// A second subscriber whose header filter matches is still allowed (competing +// consumers with a shared filter are legitimate). +func TestConflictingHeaderFiltersOnDurableRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.hfconflict", Subjects: []string{"e"}} + mk := func(region string) *pb.Source { + src := queueSource("events.hfconflict.consumer", "events.hfconflict", "e") + src.Filters = []*pb.Filter{{ + Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "region", Value: region}}, + }} + return src + } + + // Subscriber A (region=us) claims the durable and stays live; receiving a + // matching message proves it is past the claim. + outA := make(chan *pb.Message, 1) + ctxA, cancelA := context.WithCancel(ctx) + defer cancelA() + go p.Subscribe(ctxA, mk("us"), outA) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Body: []byte("k"), Headers: map[string]string{"region": "us"}})) + require.Nil(t, p.Ack(ctx, recv(t, outA).GetUuid())) + + // B tries to share the same durable with a different header filter: rejected. + perr := p.Subscribe(ctx, mk("eu"), make(chan *pb.Message, 1)) + require.NotNil(t, perr, "a conflicting header filter on a shared durable must be rejected") + assert.True(t, perr.GetIsFatal(), "the rejection is a client misconfiguration, not retriable") + assert.Contains(t, perr.GetMessage(), "header filter") + + // A matching second subscriber is accepted (does not reject). + outC := make(chan *pb.Message, 1) + ctxC, cancelC := context.WithCancel(ctx) + defer cancelC() + errC := make(chan *pb.Error, 1) + go func() { errC <- p.Subscribe(ctxC, mk("us"), outC) }() + select { + case perr := <-errC: + t.Fatalf("a matching header filter must not be rejected: %s", perr.GetMessage()) + case <-time.After(500 * time.Millisecond): + } + + // Once every subscriber of the durable ends, a fresh filter is allowed again + // (the fingerprint is forgotten when the last reference drops). + cancelA() + cancelC() + require.Eventually(t, func() bool { + return p.Subscribe(context.Background(), func() *pb.Source { + s := mk("eu") + s.DeclareOnly = true + return s + }(), nil) == nil + }, 5*time.Second, 50*time.Millisecond, + "a new filter must be accepted after the durable's subscribers all leave") +} + func TestDurableConsumerResumesBacklog(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) From 8420584e6555b85fe3f23d2e5ffde0f84a357965 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:19:37 -0400 Subject: [PATCH 43/77] fix: keep a dead-lettered message resolvable until Term succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeadLetter published the DLQ copy, then deleted the active-message entry, then Term'd the original. If the DLQ publish succeeded but Term failed, DeadLetter returned an error and the server fell back to nacking the same uuid (server.go) — but the entry was already gone, so the fallback nack could not resolve it. The original then stayed ack-pending until AckWait instead of redelivering promptly, and a later redelivery could publish a second DLQ copy once the original's dedup window had passed. Delete the entry only after Term succeeds, so a failed Term leaves the message in flight and resolvable for the fallback nack — the same keep-it-resolvable contract the ensure/publish-failure paths above already follow. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 7 ++- .../provider/connectors/natsjs/natsjs_test.go | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 8d6a8bb..0cc5fec 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1277,10 +1277,15 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid uuid, dlqMsg.Subject, perr.Error()) return &pb.Error{Message: fmt.Sprintf("dead-letter publish: %s", perr.Error())} } - bd.activeMessages.Delete(uuid) + // Term only after the DLQ copy is safely stored, and drop the active-message + // entry only after Term succeeds: if Term fails, DeadLetter returns an error + // and the server falls back to nacking this uuid (server.go), which can only + // resolve while the entry is still present. Deleting first would strand the + // original ack-pending until AckWait instead of redelivering promptly. if err := m.Term(); err != nil { return &pb.Error{Message: err.Error()} } + bd.activeMessages.Delete(uuid) return nil } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index c03aec3..1cef08d 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -346,6 +346,49 @@ func TestDeadLetterEmptyAddressKeepsMessage(t *testing.T) { require.Nil(t, p.Ack(ctx, again.GetUuid())) } +// TestDeadLetterFailedTermKeepsMessageResolvable: if the DLQ copy is stored but +// Term fails, DeadLetter must leave the active-message entry in place. The +// server resolves its fallback nack (server.go, on a DeadLetter error) by +// uuid, so deleting the entry before Term succeeds would make that nack a +// no-op and strand the original ack-pending until AckWait, instead of +// redelivering it promptly. +func TestDeadLetterFailedTermKeepsMessageResolvable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlterm", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.dlterm.consumer", "events.dlterm", "job") + src.Options["DeadLetterAddress"] = "events.dlterm.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + + // Force Term to fail without breaking the DLQ publish: ack the underlying + // message directly (marking it already-acked), so DeadLetter's DLQ publish + // still lands but its subsequent Term returns ErrMsgAlreadyAckd. + bd := p.getBrokerDetailsByIdentifier("test-client") + mu, ok := bd.activeMessages.Get(m.GetUuid()) + require.True(t, ok) + require.NoError(t, mu.(inflightMsg).msg.Ack()) + + perr := p.DeadLetter(ctx, src, m.GetUuid()) + require.NotNil(t, perr, "DeadLetter must surface the Term failure") + + // The DLQ copy was stored... + dlqStats := p.SourceStats(ctx, &pb.Source{Name: "x", Address: &pb.Address{Name: "events.dlterm.dlq"}}) + assert.Equal(t, int64(1), dlqStats.GetMessageCount(), "the DLQ copy must still have been published") + + // ...but the original stays resolvable so the server's fallback nack lands. + _, stillThere := bd.activeMessages.Get(m.GetUuid()) + assert.True(t, stillThere, + "a failed Term must not remove the active-message entry the fallback nack needs") +} + // TestPrefetchMapsToMaxAckPending: a positive PrefetchCount becomes the // consumer's MaxAckPending. A prefetch of 0 is the AMQP "unlimited" // convention — the amqp091 connector honors it by leaving the channel From 36452be2ea466a324bdc3b8211283d13049b548f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:23:30 -0400 Subject: [PATCH 44/77] fix: don't leak a NATS connection on a concurrent Connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gRPC server checks "is this client already connected?" and then calls the provider's Connect without holding a lock across the two steps (brokerConnect), so two Connect calls for one client identifier can both pass the check and enter the provider. Connect dialed a NATS connection and then unconditionally overwrote the connections-map entry, so the second call orphaned the first's *nats.Conn. Because the connector dials with MaxReconnects(-1), that orphan keeps reconnecting in the background indefinitely rather than being cleaned up. Insert under a provider mutex: if a connection for the identifier already exists, close the freshly-dialed redundant one and return success (the client already has a working connection — the same outcome the server reports for a duplicate Connect) instead of overwriting and leaking. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 19 +++++++++ .../provider/connectors/natsjs/natsjs_test.go | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 0cc5fec..97769f3 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -159,6 +159,12 @@ type natsjsProvider struct { connections *util.ConcurrentMap // streams collapses concurrent stream-creation calls across connections. streams *streamRegistry + // connectMu serializes the check-then-insert in Connect. The gRPC server's + // own "already connected?" guard (brokerConnect) is not atomic, so two + // Connect calls for one client identifier can both reach the provider; + // without this, the second would overwrite the first's entry and orphan its + // nats.Conn, which reconnects forever (MaxReconnects(-1)). + connectMu sync.Mutex } // streamRegistry collapses concurrent ensureStream calls for the same stream @@ -430,7 +436,20 @@ func (p *natsjsProvider) Connect(ctx context.Context, cfg *pb.ConnectionConfigur if nc.IsConnected() { bd.state.Store(provider.CONNECTED) } + // Insert under connectMu so a racing Connect for the same client identifier + // cannot both install a connection: the loser closes its freshly-dialed + // nats.Conn instead of overwriting the winner's and leaking a background- + // reconnecting link. A client already has a working connection in that case, + // so this is a success (mirrors the server's "connected more than once"). + p.connectMu.Lock() + if _, exists := p.connections.Get(clientID); exists { + p.connectMu.Unlock() + nc.Close() + util.Logger.Debugf("natsjs: client %s already connected; closing the redundant link", clientID) + return nil + } p.connections.Add(clientID, bd) + p.connectMu.Unlock() util.Logger.Debugf("natsjs: client %s connecting to %s", clientID, url) return nil } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 1cef08d..b11a070 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net" + "sync" "sync/atomic" "testing" "time" @@ -110,6 +111,45 @@ func TestConnectDisconnect(t *testing.T) { assert.False(t, p.ClientExists("test-client")) } +// TestConcurrentConnectClosesRedundantLink: the gRPC server's "already +// connected?" check is not atomic, so two Connect calls for one client +// identifier can both reach the provider. Only one connection may survive; the +// other's nats.Conn must be closed rather than orphaned to reconnect forever +// (MaxReconnects(-1)). The proof is the server seeing exactly one live client. +func TestConcurrentConnectClosesRedundantLink(t *testing.T) { + s := runJetStreamServer(t) + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + cfg := &pb.ConnectionConfiguration{ + Host: "127.0.0.1", + Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: "test", + } + + ctx := context.Background() + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if perr := p.Connect(ctx, cfg, false); perr != nil { + t.Errorf("connect: %s", perr.GetMessage()) + } + }() + } + wg.Wait() + require.True(t, p.WaitForConnect(ctx)) + + // Exactly one live server-side client connection: the redundant dial was + // closed. Drain can lag the close slightly, so let it settle. + require.Eventually(t, func() bool { + return s.NumClients() == 1 + }, 5*time.Second, 50*time.Millisecond, + "a racing Connect must leave exactly one live NATS connection, not leak the loser") + + p.Disconnect(ctx) +} + func TestSupportedSourceOptions(t *testing.T) { p := NewNATSJetStreamProvider().(*natsjsProvider) opts := p.SupportedSourceOptions() From 4f5845c72e775ef2602a7b811fbff2eb49569333 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:25:18 -0400 Subject: [PATCH 45/77] docs: note competing consumers on a durable must share a header filter Document the constraint the header-filter rejection introduces: because header filtering is per consumer proxy-side, two competing subscribers of one durable that disagree on their header filter would each drop the other's messages, so a second live subscriber whose header filter differs is rejected. Extends the existing proxy-side header-filter limitation bullet. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 10eaa18..6254d91 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -436,7 +436,14 @@ amqp091 connector, so existing client sources validate unchanged: therefore scale with the subject-matched traffic, not the header-matched traffic. That is fine when header filters refine an already-narrow subject, but a high-volume address consumed almost entirely through header filters - should be remodeled onto routing keys (subjects) instead. + should be remodeled onto routing keys (subjects) instead. Because this + filtering is per consumer, competing subscribers of one durable must share + the same header filter: a RabbitMQ headers binding is queue-wide, but here + each consumer applies only its own filter, so a message the server hands to + the "wrong" subscriber would be dropped rather than reaching the one that + wanted it. A second live subscriber whose header filter differs from the + durable's is therefore rejected (subject filters, which the server enforces, + still update the shared consumer in place). - **Dead-letter is a proxy-side republish.** The DLQ is an ordinary stream fed by the connector; there is no advisory-driven re-consumption. A failed DLQ publish fails the dead-letter call — the message stays in flight and is From 769ecdba9b9077735c18d9fed3ecc1e806ed8842 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 14 Jul 2026 19:52:39 -0400 Subject: [PATCH 46/77] chore: satisfy the strict CI golangci-lint config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI go-lint job runs golangci-lint over the whole module with .github/linters/golangci-config.yaml, which enables dupl, gocognit, gocyclo, gocritic, gosec, and unparam on top of the default set. Those findings had accumulated unnoticed because they only surface under that config and whole-module scope, not a per-package run with defaults. Resolve them so the job is green: - unparam: drop takeMessage's unused *natsBrokerDetails return and update its Ack/Nack/Retry callers. - gocritic (unlambda): pass subjectSubsumes directly instead of wrapping it in a lambda in the test. - gocognit/gocyclo: nolint on Subscribe, matching how amqp091's queueSubscribe/declareBinding annotate the same one-linear-path shape. - gosec (G101): nolint on tokenEscapeTriggers — a set of NATS subject reserved characters, not a credential. - dupl: nolint on the two address-coexistence tests, kept deliberately parallel for readability. No behavior change. Signed-off-by: Michael Otteni --- .../provider/connectors/natsjs/helpers.go | 2 +- .../connectors/natsjs/helpers_test.go | 2 +- internal/provider/connectors/natsjs/natsjs.go | 19 ++++++++++--------- .../provider/connectors/natsjs/natsjs_test.go | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index c2db15b..371a71a 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -79,7 +79,7 @@ var tokenUnescapes = map[byte]byte{ // tokenEscapeTriggers are the characters that force a token into the escaped // form; a token free of them passes through literally. -const tokenEscapeTriggers = "~ \t\r\n\f*>#" +const tokenEscapeTriggers = "~ \t\r\n\f*>#" //nolint:gosec // G101 false positive: NATS subject reserved characters, not a credential // escapeToken maps one address or routing-key token onto a NATS-legal // subject token, injectively: distinct tokens never yield the same output. A diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index b8a86c1..0bfa007 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -302,7 +302,7 @@ func TestDirectFilterSubjectsAreExact(t *testing.T) { } func TestSubjectSubsumes(t *testing.T) { - cover := func(wide, narrow string) bool { return subjectSubsumes(wide, narrow) } + cover := subjectSubsumes assert.True(t, cover("a.>", "a.b")) assert.True(t, cover("a.>", "a.b.c")) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 97769f3..cb11aad 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -619,8 +619,9 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, // Subscribe ensures topology, creates a consumer, and forwards deliveries to the // message channel until the context is cancelled. Blocks per the Provider -// contract. -func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out chan<- *pb.Message) *pb.Error { +// contract. It mirrors amqp091's queueSubscribe: one linear setup path with +// many small guards (hence the complexity nolint). +func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out chan<- *pb.Message) *pb.Error { //nolint:gocognit,gocyclo bd, err := p.getBrokerDetails(ctx) if err != nil { return &pb.Error{Message: err.Error(), IsFatal: true} @@ -1158,21 +1159,21 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai } } -func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstream.Msg, *natsBrokerDetails, *pb.Error) { +func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstream.Msg, *pb.Error) { bd, err := p.getBrokerDetails(ctx) if err != nil { - return nil, nil, &pb.Error{Message: err.Error()} + return nil, &pb.Error{Message: err.Error()} } mu, ok := bd.activeMessages.Get(uuid) if !ok { - return nil, bd, &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} + return nil, &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} } bd.activeMessages.Delete(uuid) - return mu.(inflightMsg).msg, bd, nil + return mu.(inflightMsg).msg, nil } func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { - m, _, perr := p.takeMessage(ctx, uuid) + m, perr := p.takeMessage(ctx, uuid) if perr != nil { return perr } @@ -1184,7 +1185,7 @@ func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { // Nack negatively acknowledges for immediate redelivery (literal AMQP nack). func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { - m, _, perr := p.takeMessage(ctx, uuid) + m, perr := p.takeMessage(ctx, uuid) if perr != nil { return perr } @@ -1199,7 +1200,7 @@ func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { // retry-queue idiom; JetStream increments the delivery count for us, which // handleDelivery surfaces as x-retry-count. func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, delay int32) *pb.Error { - m, _, perr := p.takeMessage(ctx, uuid) + m, perr := p.takeMessage(ctx, uuid) if perr != nil { return perr } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index b11a070..3622400 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1142,7 +1142,7 @@ func TestPrefixAddressesCoexist(t *testing.T) { // both onto one JetStream stream, whose config each address's re-ensure then // flips to its own subjects — each breaking the other. The name mapping must // keep them on separate streams. -func TestUnderscoreAddressesCoexist(t *testing.T) { +func TestUnderscoreAddressesCoexist(t *testing.T) { //nolint:dupl // deliberately parallel to TestEscapedAddressesCoexist s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -1173,7 +1173,7 @@ func TestUnderscoreAddressesCoexist(t *testing.T) { // root "evt._.b" — one shared subject space and one shared stream, so each // address received the other's messages and each ensure reconfigured the // other's stream. Escaped tokens must keep them fully apart. -func TestEscapedAddressesCoexist(t *testing.T) { +func TestEscapedAddressesCoexist(t *testing.T) { //nolint:dupl // deliberately parallel to TestUnderscoreAddressesCoexist s := runJetStreamServer(t) p, ctx := connectClient(t, s) From 0786c9abeb44a4d3931519047ad9ff1ef1056e93 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 15 Jul 2026 01:07:24 -0400 Subject: [PATCH 47/77] build: repin the integration module's indirect deps The integration tests are a separate module that replaces the root one, so it carries its own copy of the root's dependency graph. Adding nats-server as a test dependency moved golang.org/x/sys to 0.46.0 and x/term to 0.44.0 in the root go.mod but left this module pinned to the versions before, which is enough for the whole module to refuse to build: go: updates to go.mod needed; to update it: go mod tidy CI never caught it because it builds the root module only; nothing runs `go test -tags=integration`. Run go mod tidy so the suite compiles again. Signed-off-by: Michael Otteni --- tests/integration/go.mod | 4 ++-- tests/integration/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/go.mod b/tests/integration/go.mod index e856be6..98cc1e1 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -50,8 +50,8 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index 1278f48..ad97764 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -127,10 +127,10 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= From 377a6d7417ffb7fe3d7a9154921b5db7dace222a Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 15 Jul 2026 01:07:41 -0400 Subject: [PATCH 48/77] test: give TestConsumeContinueOffset's second consumer its own connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test runs two consumers in sequence over one gRPC connection and one done channel, and both are shared in ways that decide the result: - Arke identifies a client by name and peer address, so a second ConsumerClient over the same connection is the *same* Arke connection. The first consumer's deferred Disconnect (in consumeMessages) therefore tears down the connection the second one is using, and its Consume stream ends with EOF. - consumeMessages signals its own exit on done, so the first consumer's exit breaks the second consumer's receive loop. Both are races the test happens to win against RabbitMQ, whose two disconnects land before the second consumer re-connects, and loses against a provider that tears down at any other speed — measuring 0, 1 or 3 of the 10 messages it published depending on timing, with nothing to say that the offset behaviour it is supposed to be testing was ever exercised. Give the second consumer its own connection and its own done channel so what it counts is its own. Signed-off-by: Michael Otteni --- tests/integration/integration_test.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 1ee0480..11d3124 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -821,13 +821,22 @@ func TestConsumeContinueOffset(t *testing.T) { c.Disconnect(ctx, &pb.Empty{}) expectedMessageCount = 10 - c2 := pb.NewConsumerClient(consumerConnection) + // The second consumer needs its own gRPC connection and its own done + // channel. Arke identifies a client by name and peer address, so a second + // ConsumerClient over the same connection is the *same* Arke connection, + // and the first consumer's deferred Disconnect would tear this one down. + // consumeMessages also signals its own completion on done, so sharing that + // channel lets the first consumer's exit break this loop. + consumerConnection2 := connect() + defer consumerConnection2.Close() + done2 := make(chan bool) + c2 := pb.NewConsumerClient(consumerConnection2) ctx2, cancel2 := context.WithTimeout(context.Background(), 15*time.Second) defer cancel2() defer c2.Disconnect(ctx2, &pb.Empty{}) messages2 := make(chan *pb.Message) - go consumeMessages(consumerConnection, c2, ctx2, messages2, done, clientConnected, source, defaultHandler, t) + go consumeMessages(consumerConnection2, c2, ctx2, messages2, done2, clientConnected, source, defaultHandler, t) <-clientConnected err = mf.ProduceMessagesUnaryWOConnect(pctx, pc, expectedMessageCount, message, false) @@ -839,7 +848,7 @@ func TestConsumeContinueOffset(t *testing.T) { select { case <-messages2: msgCount2++ - case <-done: + case <-done2: breakLoop = true case <-time.After(5 * time.Second): breakLoop = true From e34b2522e2fc5493c5350e8f6933c22fd58278ff Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 15 Jul 2026 01:08:48 -0400 Subject: [PATCH 49/77] fix: close the parity gaps Arke's integration suite found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running tests/integration against a JetStream server (the suite takes its provider from ARKE_BROKER_TYPE, so it needs no changes to do this) puts the connector against the same assertions amqp091 answers. Six divergences, most of them silent: A plain nack redelivered forever. A nack on this contract is a rejection: amqp091 answers it with Delivery.Nack(requeue=false), so RabbitMQ drops the message, or moves it to the queue's dead-letter exchange — which the server asks for separately through DeadLetter. JetStream's Nak means redeliver now, so one nacked message became an unbounded loop between server and client, measured at 30,467 deliveries in five seconds, because a client that rejected a message rejects each redelivery straight back. Map a nack to Term, which stops redelivery, and leave redelivery to Retry. The exception is the server's fallback nack after a failed DeadLetter, whose whole purpose is to put the message back for another attempt: DeadLetter marks the message while its attempt is unresolved, and a nack of a marked message naks. Offset "continue" replayed the log. It means resume where this source stopped, which needs a position the broker kept between subscriptions. amqp091 reads one from RabbitMQ Streams' server-side offset tracking, keyed by consumer name; the equivalent is a durable, but a group-less stream source was ephemeral, so "continue" fell back to its first-creation position — the whole log — every time. Give a source asking for it a durable named after itself. Other offsets position the reader from the log and stay ephemeral, so same-named readers still read independently. A source with no bindings received everything. An empty subject list became the empty pattern, which maps to a catch-all, where amqp091 declares no binding at all and the source receives nothing. A consumer must carry one filter subject, so express nothing as a subject the token escaping can never emit. Three cases are deliberately not that: an explicit "" key is a literal and selects the empty routing key only; a headers address with filters selects the whole address, because a headers exchange ignores routing keys and the filter decides; and a stream source is not bound to its address at all — amqp091 reads a stream by name and never binds — so it reads the whole log. That last one is not an edge case: a stream source on a stream address with no subjects is the shape of an ordinary multi-tenant bootstrap consumer. Address-to-address binding did nothing. ParentAddress had no implementation and no error: what was published to the parent never reached the child's consumers. On a broker that routes it is a routing rule; here the stream is the storage, so the child's stream sources the bound subjects out of the parent's, keeping each message's subject — only one stream ever listens on a subject, and a routed-in message is indistinguishable from one published to the child directly. Bindings accumulate rather than replace, as they do on RabbitMQ. Offsets and stream stats were JetStream's, not Arke's. An offset counts from 0 in the vocabulary amqp091 established; a JetStream sequence counts from 1, and reporting one while accepting the other back skips a message per round-trip. Convert in both directions. A stream source also reported a queue's message count, which its readers do not share, and answered an empty log with silence where amqp091 says "Offset not found". Error strings a client may match on now read identically on both connectors. Two publish combinations were accepted that amqp091 refuses: dedup on a queue address (RabbitMQ gets dedup from streams, so a client writing this would silently lose it there) and Confirm on the fire-and-forget streaming publish. Refuse both, with amqp091's wording. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 148 ++++++-- .../provider/connectors/natsjs/helpers.go | 154 ++++++-- .../connectors/natsjs/helpers_test.go | 18 +- internal/provider/connectors/natsjs/natsjs.go | 280 ++++++++++++-- .../provider/connectors/natsjs/natsjs_test.go | 343 +++++++++++++++++- 5 files changed, 847 insertions(+), 96 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 6254d91..a8dcad1 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -112,6 +112,45 @@ filters to the widest set before creating the consumer; the surviving wildcard filter matches everything the dropped filters did. Intersecting patterns where neither contains the other are kept as-is. +A source whose address carries *no* subjects declares no bindings and so +receives nothing, matching amqp091 (which binds nothing at all in that case). +A JetStream consumer must carry at least one filter subject, so "nothing" is +expressed as a filter on a subject no published subject can contain — a token +the escaping above can never emit. Three cases are deliberately not that: + +- an empty binding key (`""`, as distinct from no keys) is a literal, and + selects the empty routing key only, exactly as it matches on a topic or + direct exchange; +- a `STREAM` source is not bound to its address at all — amqp091 reads a + RabbitMQ stream by name and never declares a binding — so it reads the + whole log; +- a headers address with filters selects the whole address, because a headers + exchange ignores routing keys and the header match decides. amqp091 binds a + single `""` key there purely to have somewhere to hang the header + arguments; `evaluateFilters` is this connector's stand-in for those. + +### Address-to-address binding + +An address may name a `ParentAddress`, binding it to that parent: what is +published to the parent under the child's binding keys is routed on to the +child, and reaches the child's own consumers. On a broker that routes, this is +a routing rule. Here the stream *is* the storage, so the equivalent is to +source the bound subjects out of the parent's stream into the child's +(`ensureStreamFor`), keeping the subject each message was published under +rather than transforming it. That means only the parent's stream ever listens +on the parent's subjects — no two streams claim one subject space — while a +message routed in from the parent is indistinguishable, to a consumer, from +one published to the child directly. The child's consumers filter on both +their address's subjects and the bound parent subjects. + +The binding keys are the child's own subjects, matched by the *parent* +exchange's type, so they translate exactly like any other binding. Bindings +accumulate: declaring one never removes another, as on RabbitMQ, so a second +subscriber binding a different key on the same address adds to the set rather +than replacing it. Nothing unbinds — a binding outlives the subscription that +declared it, as an AMQP exchange-to-exchange binding outlives the client that +bound it. + This subject scheme — and the stream/durable name encoding above — is part of the connector's persistence contract: retained messages are stored under these subjects for the life of the stream's retention limits, so any future @@ -151,7 +190,13 @@ The connector chooses the consumer kind from the source for per-instance, transient subscriptions — and for `STREAM` sources without a `ConsumerGroup`, whose subscribers are independent readers of the shared log, each positioned by its own `Offset` (RabbitMQ stream - consumers do not compete). They are also deleted eagerly + consumers do not compete). The exception is a group-less `STREAM` source + asking for `Offset: continue`, which is a request to resume where that + source last stopped and so needs a position the broker keeps between + subscriptions: it gets a durable named after the source, the way amqp091 + answers `continue` from RabbitMQ Streams' server-side offset tracking + (keyed by consumer name). Every other offset positions the reader from the + log itself and stays ephemeral. They are also deleted eagerly when their subscription ends (or never starts — a declare-only call, or a failure to begin consuming), so the threshold only has to cover unclean exits and churning transient clients cannot accumulate dead consumers @@ -191,20 +236,29 @@ channel's unacked deliveries immediately; without the nak they would only redeliver after the full ack wait. The `DeliverPolicy` is taken from the `Offset` option, mirroring the amqp091 -connector's offset vocabulary so both accept the same values: `first`/`continue` --> deliver all, `last` -> the final message, `next` (or unset) -> deliver new, -and an absolute number -> start at that stream sequence. Any other value fails -the subscribe (as in amqp091's offset parsing) rather than silently starting -the consumer at a different position than it asked for. The offset only -applies on first creation: JetStream fixes a durable's start position when the -consumer is created, so a re-subscribe that requests a different offset logs a -warning and resumes from the durable's stored ack position — use a new durable -(source or `ConsumerGroup` name) to reposition. Only that start-position -conflict is absorbed; any other error creating or updating the consumer fails -the subscribe, since resuming a consumer whose configuration silently differs -from the requested one would consume the wrong way with no signal. Numeric -offsets are JetStream stream sequence numbers (as surfaced by `SourceStats`) -and are not portable across brokers. +connector's offset vocabulary so both accept the same values: `first` -> +deliver all, `last` -> the final message, `next` (or unset) -> deliver new, +`continue` -> resume from the position the source's durable holds (deliver all +on its first creation, when there is no position yet — RabbitMQ answers a +`continue` with no stored offset the same way), and an absolute number -> +start at that offset. Any other value fails the subscribe (as in amqp091's +offset parsing) rather than silently starting the consumer at a different +position than it asked for. The offset only applies on first creation: +JetStream fixes a durable's start position when the consumer is created, so a +re-subscribe that requests a different offset logs a warning and resumes from +the durable's stored ack position — use a new durable (source or +`ConsumerGroup` name) to reposition. Only that start-position conflict is +absorbed; any other error creating or updating the consumer fails the +subscribe, since resuming a consumer whose configuration silently differs from +the requested one would consume the wrong way with no signal. + +A numeric offset counts from 0, naming a message's position in the log the way +a RabbitMQ Stream offset does — not the raw JetStream sequence, which counts +from 1. The connector converts in both directions, so an offset read from +`SourceStats` and handed back as a source's `Offset` names the same message it +did on either broker. The value is still not portable *across* brokers: offset +7 is the eighth message of whichever log is being read, and two brokers' logs +hold different messages. ## Ack, retry, and dead-letter mapping @@ -216,7 +270,21 @@ connector maps those onto JetStream primitives: | ack | `Ack` | `msg.Ack()` | | nack, `requeue_delay > 0` | `Retry(delay)` | `msg.NakWithDelay(delay)` | | nack, `requeue_delay == 0`, DLA set | `DeadLetter` | publish to DLQ + `Term()` | -| nack, `requeue_delay == 0`, no DLA | `Nack` | `msg.Nak()` | +| nack, `requeue_delay == 0`, no DLA | `Nack` | `msg.Term()` | + +Note which primitive a plain nack maps to. A nack on this contract is a +rejection, not a requeue: amqp091 answers the same call with +`Delivery.Nack(requeue=false)`, so RabbitMQ drops the message, or moves it to +the queue's dead-letter exchange — which the server here asks for explicitly +through `DeadLetter` instead. JetStream's `Nak` means the opposite, redeliver +now, so using it for a nack turns a single nacked message into an unbounded +delivery loop: a client that rejected a message rejects each redelivery +straight back, at thousands of deliveries a second for as long as the +subscription lives. `Term` carries the intended meaning — stop redelivering — +and `Retry` remains the way to ask for redelivery. The one exception is the +server's fallback nack after a failed `DeadLetter` (below), whose purpose is +to put the message back so dead-lettering can be retried; `DeadLetter` marks +the message before returning that error, and a nack of a marked message naks. A delivered message that is neither acked nor nacked redelivers after the consumer's ack wait (`NATSJS_ACK_WAIT`, default 30s). That value sets one @@ -378,8 +446,11 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Retry-count header (`x-retry-count`) | Proxy | Synthesized from JetStream `NumDelivered`. | | Dead-letter (DLX) | Proxy | No native DLX; republish to DLQ subject then `Term()`. | | Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters` (see limitations). | +| Address-to-address binding (`ParentAddress`) | Native | The child's stream sources the bound subjects from the parent's, keeping each message's subject. | | `MessageTTL` / `Expires` | Partial | Mapped onto stream-level `MaxAge` (see limitations). | +| Dead-letter on message expiry | Drop | RabbitMQ expires a message off a queue into its DLX; per-source TTL is not applied here, so nothing expires per source to dead-letter (see limitations). | | Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | +| Max message size | Partial | NATS caps a payload at the server's `max_payload` (1MB default) where RabbitMQ's `max_message_size` default is 16MB (see limitations). | | Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | | RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | @@ -395,23 +466,41 @@ amqp091 connector, so existing client sources validate unchanged: | `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. A warning is logged at subscribe time. | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing-key override for dead-lettered messages; when unset, the copy keeps the message's original routing key (RabbitMQ's default dead-letter behavior). | -| `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a numeric sequence). | +| `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a number counting from 0, as reported by `SourceStats`). | | `ConsumerGroup` | string | Durable consumer group name (stream sources; also names the durable that single-active instances coordinate through, and is required for a single-active stream source). | ## Known limitations -- **Per-source TTL fidelity.** The connector uses one stream per address - root, so a per-source `MessageTTL` / `Expires` cannot be mapped onto the - shared stream without one source's value flapping another's retention. - Both options are therefore accepted but not applied, and a subscribe that - sets either logs a warning naming the source — silent divergence in data - retention is the one place a client must not have to read a design doc to - notice. Retention comes from the stream-wide `NATSJS_STREAM_MAX_AGE` / - `NATSJS_STREAM_MAX_BYTES` configuration. True per-source TTL (or switching - queue sources to a delete-on-ack policy such as `WorkQueuePolicy` / - `InterestPolicy`) needs a +- **Per-source TTL fidelity, and expiry-driven dead-lettering.** The + connector uses one stream per address root, so a per-source `MessageTTL` / + `Expires` cannot be mapped onto the shared stream without one source's + value flapping another's retention. Both options are therefore accepted but + not applied, and a subscribe that sets either logs a warning naming the + source — silent divergence in data retention is the one place a client must + not have to read a design doc to notice. Retention comes from the + stream-wide `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` + configuration. True per-source TTL (or switching queue sources to a + delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a per-source stream topology, and `Retention` is immutable, so that is a stream-recreate migration rather than an in-place change. + + A consequence worth stating on its own: RabbitMQ's per-queue TTL doubles as + a routing mechanism, expiring a message off its queue and *into* the + queue's dead-letter exchange, which is the classic "unprocessed after N + seconds, send it to the DLQ" idiom. Since no per-source TTL is applied + here, nothing expires per source and so nothing is dead-lettered by expiry; + a message is dead-lettered only when a client nacks it with a dead-letter + address set. Retention limits do eventually evict a message, but eviction + deletes it — there is no expiry hook to route from. Clients relying on + expiry-to-DLQ need an explicit nack, or the per-source topology above. +- **Payloads above the server's `max_payload`.** NATS enforces a maximum + message size server-side (`max_payload`, 1MB by default, 64MB hard + ceiling), where RabbitMQ's `max_message_size` defaults to 16MB. A message + in between publishes on RabbitMQ and is rejected here with `maximum payload + exceeded`. This is a server setting the connector cannot negotiate around, + so a deployment carrying large messages has to raise `max_payload` to cover + them (NATS advises staying at or below 8MB, since a large message is held + whole in memory on every hop) or move the payload out of the message. - **Publish / deliver rates are sampled, not native.** JetStream exposes absolute counters, not rates, so `SourceStats` differences them between successive calls: the first call after a (re)connect returns zero, the @@ -423,7 +512,10 @@ amqp091 connector, so existing client sources validate unchanged: limits, so its depth keeps growing after consumers catch up, which would mislead anything using message count as queue length (e.g. consumer autoscaling). Sources without a durable consumer fall back to the stream - view. A durable source's consumer count is likewise per source: the number + view. A `STREAM` source reports no message count at all, as on amqp091: + the readers of one retained log have no common backlog, and their reading + is the offset pair instead. A stream with no offset to report answers with + RabbitMQ's own `Offset not found` error rather than a silent zero. A durable source's consumer count is likewise per source: the number of clients with an open pull request on its consumer — a client working through a full pull buffer can briefly read as zero — while sources without a durable consumer report the stream-wide consumer count, which diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 371a71a..9439086 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -336,20 +336,63 @@ func streamSubjectsFor(addressName string) []string { } // filterSubjectsFor maps a source's routing-key patterns to NATS consumer -// filter subjects. An empty pattern set means everything under the address. -// A pattern whose AMQP '#' tail became '>' also gets the zero-word variant: -// AMQP '#' matches zero or more words while NATS '>' matches one or more, so -// binding "a.#" must match routing key "a" too. +// filter subjects. +// +// A source whose address has a parent also selects the subjects its address is +// bound to in the *parent's* space: messages routed in from the parent are +// sourced into this address's stream keeping the subject they were published +// under (see parentBindingSubjects and ensureStreamFor). func filterSubjectsFor(source *pb.Source) []string { + own := ownFilterSubjectsFor(source) + parents := parentBindingSubjects(source.GetAddress()) + if len(parents) == 0 { + return own + } + return pruneSubsumed(append(own, parents...)) +} + +// parentBindingSubjects gives the subjects an address's bindings select in its +// parent address's subject space, or nil if it has no parent. +// +// An address-to-address binding uses the child's own subjects as the binding +// keys (amqp091's declareExchange binds each of them from the child exchange +// to the parent), and those keys are matched by the *parent* exchange's type — +// so the mapping is exactly the ordinary binding translation, run against the +// parent's name and type. With no subjects there are no bindings and so +// nothing is routed in, which is why this returns nil rather than asking +// ownFilterSubjectsFor for an unmatchable subject. +func parentBindingSubjects(addr *pb.Address) []string { + parent := addr.GetParentAddress() + if parent.GetName() == "" || len(addr.GetSubjects()) == 0 { + return nil + } + return ownFilterSubjectsFor(&pb.Source{ + Address: &pb.Address{ + Name: parent.GetName(), + Type: parent.GetType(), + Subjects: addr.GetSubjects(), + }, + }) +} + +// ownFilterSubjectsFor maps a source's routing-key patterns to the filter +// subjects of its own address. A pattern whose AMQP '#' tail became '>' also +// gets the zero-word variant: AMQP '#' matches zero or more words while NATS +// '>' matches one or more, so binding "a.#" must match routing key "a" too. +// +// A source with no patterns declares no bindings and so receives nothing — +// see boundNothingSubjects for the exceptions and for how "nothing" is +// expressed to a consumer that must carry at least one filter. +func ownFilterSubjectsFor(source *pb.Source) []string { + if len(source.GetAddress().GetSubjects()) == 0 { + return boundNothingSubjects(source) + } if source.GetAddress().GetType() == pb.Address_QUEUE { return directFilterSubjectsFor(source) } prefix := subjectPrefix(source.GetAddress().GetName()) pats := source.GetAddress().GetSubjects() - if len(pats) == 0 { - pats = []string{""} - } out := make([]string, 0, len(pats)+1) seen := make(map[string]bool, len(pats)+1) add := func(s string) { @@ -362,8 +405,9 @@ func filterSubjectsFor(source *pb.Source) []string { pat := translateWildcards(p) switch { case pat == "": + // An empty binding key is a literal, not a wildcard: on a topic + // exchange it matches the empty routing key and nothing else. add(prefix) - add(prefix + ".>") case strings.HasSuffix(pat, ">"): base := strings.TrimSuffix(strings.TrimSuffix(pat, ">"), ".") if base == "" { @@ -376,16 +420,29 @@ func filterSubjectsFor(source *pb.Source) []string { add(prefix + "." + pat) } } - // An AMQP binding set may be redundant — "orders.#" alongside - // "orders.created" — which is harmless on a broker that routes a message - // to a queue once no matter how many of its bindings match. JetStream - // instead rejects a consumer whose filter subjects overlap (one being a - // subset of another), so drop every filter another filter already covers. - // The surviving wider filter matches everything the dropped one did. - kept := make([]string, 0, len(out)) - for _, s := range out { + return pruneSubsumed(out) +} + +// pruneSubsumed drops every subject another subject in the set already covers. +// An AMQP binding set may be redundant — "orders.#" alongside "orders.created" +// — which is harmless on a broker that routes a message to a queue once no +// matter how many of its bindings match. JetStream instead rejects a consumer +// whose filter subjects overlap (one being a subset of another). The surviving +// wider filter matches everything the dropped one did. Duplicates are dropped +// too, since a subject trivially covers itself. +func pruneSubsumed(subjects []string) []string { + uniq := make([]string, 0, len(subjects)) + seen := make(map[string]bool, len(subjects)) + for _, s := range subjects { + if !seen[s] { + seen[s] = true + uniq = append(uniq, s) + } + } + kept := make([]string, 0, len(uniq)) + for _, s := range uniq { subsumed := false - for _, t := range out { + for _, t := range uniq { if t != s && subjectSubsumes(t, s) { subsumed = true break @@ -398,12 +455,49 @@ func filterSubjectsFor(source *pb.Source) []string { return kept } -func directFilterSubjectsFor(source *pb.Source) []string { +// unmatchableToken is a subject token no published subject can contain, so a +// filter subject ending in it selects nothing while still being a legal subset +// of the address's stream — which is how a consumer, obliged to carry at least +// one filter, expresses "bound to nothing". +// +// It is unreachable by construction: escapeToken emits either a token free of +// reserved characters (never starting with '~', since '~' is itself reserved), +// or "~e" for the empty token, or '~' followed by one or more *two*-character +// escape codes — three characters at minimum. No output is ever a '~' plus a +// single character other than "~e". +const unmatchableToken = "~x" + +// boundNothingSubjects gives the filter subjects for a source whose address +// carries no routing-key patterns. amqp091 declares no binding at all in that +// case (declareBinding iterates an empty subject list), so the source receives +// nothing — with two exceptions that both mean "the whole address". +func boundNothingSubjects(source *pb.Source) []string { prefix := subjectPrefix(source.GetAddress().GetName()) + whole := []string{prefix, prefix + ".>"} + // A stream source is not bound to its address at all. amqp091 reads a + // RabbitMQ stream by name — streamSubscribe goes straight to the stream + // connection and never declares an exchange or a binding — so its reader + // sees the whole log. Selecting everything the address's stream captures + // is the equivalent. This is not a corner case: a Source_STREAM on an + // Address_STREAM with no subjects is the ordinary way to read a stream, + // so reading it as "no bindings" would silently deliver nothing. + if source.GetType() == pb.Source_STREAM { + return whole + } + // A headers address with filters: amqp091 binds a single "" key purely to + // have a binding to hang the header arguments on (declareBinding fakes the + // subject in when the address carries none). A headers exchange ignores + // routing keys, so that binding matches every message and the header match + // decides. evaluateFilters is this connector's stand-in for those + // arguments, so select the whole address and let it do the deciding. + if source.GetAddress().GetType() == pb.Address_FILTER && len(source.GetFilters()) > 0 { + return whole + } + return []string{prefix + "." + unmatchableToken} +} + +func directFilterSubjectsFor(source *pb.Source) []string { pats := source.GetAddress().GetSubjects() - if len(pats) == 0 { - return []string{prefix} - } out := make([]string, 0, len(pats)) seen := make(map[string]bool, len(pats)) for _, p := range pats { @@ -489,12 +583,30 @@ func durableName(source *pb.Source) string { if cg := source.GetOptions()["ConsumerGroup"]; cg != "" { return streamNameFor(cg) } + // A group-less stream source is ordinarily an independent ephemeral + // reader, but "continue" asks to resume where this source left off, + // which only a position the broker kept between subscriptions can + // answer. amqp091 reads one from RabbitMQ Streams' server-side offset + // tracking, keyed by the consumer name; JetStream keeps a position per + // durable consumer, so name a durable after the source. Every other + // offset positions the reader from the log itself and stays ephemeral, + // so same-named readers still read independently. + if wantsStoredPosition(source) { + return streamNameFor(source.GetName()) + } case pb.Source_TEMPORARY: // Transient; already returned above via the TEMPORARY guard. } return "" } +// wantsStoredPosition reports whether a source's Offset asks to resume from a +// position the broker remembers, rather than one derived from the log itself. +// Only "continue" does. +func wantsStoredPosition(source *pb.Source) bool { + return strings.EqualFold(strings.TrimSpace(source.GetOptions()["Offset"]), "continue") +} + // pbToNatsHeader converts Arke's flat string headers to a NATS header. func pbToNatsHeader(in map[string]string) nats.Header { if len(in) == 0 { diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 0bfa007..d34e47d 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -227,10 +227,17 @@ func TestFilterSubjectsFor(t *testing.T) { return &pb.Source{Address: &pb.Address{Name: "events.orders", Subjects: subjects}} } - // no patterns -> everything under the address, including the empty key + // No patterns -> no bindings -> nothing selected, like amqp091 declaring + // no binding at all. The consumer still needs one filter subject, so it + // gets an unmatchable one. assert.Equal(t, - []string{"events.orders.~", "events.orders.~.>"}, + []string{"events.orders.~." + unmatchableToken}, filterSubjectsFor(src())) + // An empty pattern is a literal empty routing key on a topic address, not + // a catch-all. + assert.Equal(t, + []string{"events.orders.~"}, + filterSubjectsFor(src(""))) // plain patterns map one to one assert.Equal(t, []string{"events.orders.~.created", "events.orders.~.region.*.updated"}, @@ -296,9 +303,14 @@ func TestDirectFilterSubjectsAreExact(t *testing.T) { assert.Equal(t, []string{"events.direct.~.created", "events.direct.~.region.us"}, filterSubjectsFor(src("created", "region.us", "created"))) + // No subjects binds nothing, exactly as on a direct exchange. assert.Equal(t, - []string{"events.direct.~"}, + []string{"events.direct.~." + unmatchableToken}, filterSubjectsFor(src())) + // An explicit empty binding key selects the empty routing key only. + assert.Equal(t, + []string{"events.direct.~"}, + filterSubjectsFor(src(""))) } func TestSubjectSubsumes(t *testing.T) { diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index cb11aad..33a050b 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "slices" "sort" "strconv" "strings" @@ -41,6 +42,18 @@ const providerName = "natsjs" // x-death equivalent. const retryCountHeaderName = "x-retry-count" +// Error strings a client may match on, so they have to read identically +// whichever connector answers. Both take amqp091's wording: noMessageError is +// what it returns for an ack, nack, or dead-letter naming a message it does +// not hold, and offsetNotFoundError is the RabbitMQ stream client's answer +// when a stream has no offset to report, which its SourceStats passes through. +const ( + noMessageError = "No message with uuid %s" + offsetNotFoundError = "Offset not found" + unsupportedConfirmError = "Unsupported: Publish does not support publish confirmation" + queueDedupError = "Message deduplication is not supported by Queue types" +) + const ( // defaultAckWait is how long the server waits for an ack before // redelivering a message. See ackWait for the trade-off it sets; @@ -153,6 +166,11 @@ type durableFilterUse struct { type inflightMsg struct { msg jetstream.Msg subKey string + // redeliverOnNack inverts Nack's terminate-by-default behaviour for this + // message. It is set only by a failed DeadLetter, whose caller answers the + // error with a nack that has to put the message back for another + // dead-letter attempt rather than destroy it. See Nack. + redeliverOnNack bool } type natsjsProvider struct { @@ -165,6 +183,9 @@ type natsjsProvider struct { // without this, the second would overwrite the first's entry and orphan its // nats.Conn, which reconnects forever (MaxReconnects(-1)). connectMu sync.Mutex + // bindMu serializes the read-modify-write of an address's + // address-to-address binding set. See ensureStreamFor. + bindMu sync.Mutex } // streamRegistry collapses concurrent ensureStream calls for the same stream @@ -505,17 +526,41 @@ func (p *natsjsProvider) Disconnect(ctx context.Context) { // (jsAPITimeout) and is detached from the triggering caller's cancellation: // the stream is shared topology and other callers may be waiting on the same // result. +// ensureStream asserts the stream for a bare address name — one with no +// address-to-address binding to declare. See ensureStreamFor. func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails, addressName string) (string, error) { - streamName := streamNameFor(addressName) - if _, ok := bd.knownStreams.Get(streamName); ok { - return streamName, nil + return p.ensureStreamFor(ctx, bd, &pb.Address{Name: addressName}) +} + +// ensureStreamFor asserts the stream backing an address, including the +// address-to-address binding its ParentAddress declares, if any. +// +// A binding from a parent address is a routing rule on a broker that routes; +// here, where the stream *is* the storage, the equivalent is to source the +// bound subjects out of the parent's stream into this one. The sourced copies +// keep the subject they were published under (no subject transform), so a +// message routed in from the parent is indistinguishable to a consumer from +// one published to this address directly, and only the parent's stream listens +// on the parent's subjects — no two streams ever claim the same subject space. +func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDetails, addr *pb.Address) (string, error) { + streamName := streamNameFor(addr.GetName()) + bindings := parentBindingSubjects(addr) + // The per-connection memo only remembers that the stream exists, which is + // all an unbound address needs. A binding set is per-address state that + // another subscriber may have extended since, so an address with a parent + // re-asserts it every time; the assertion is idempotent, and addresses + // with parents are rare enough for the extra round-trip not to matter. + if len(bindings) == 0 { + if _, ok := bd.knownStreams.Get(streamName); ok { + return streamName, nil + } } - err := p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), func() error { + build := func() error { cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jsAPITimeout()) defer cancel() - _, err := bd.js.CreateOrUpdateStream(cctx, jetstream.StreamConfig{ + cfg := jetstream.StreamConfig{ Name: streamName, - Subjects: streamSubjectsFor(addressName), + Subjects: streamSubjectsFor(addr.GetName()), Storage: jetstream.FileStorage, Retention: jetstream.LimitsPolicy, Duplicates: defaultDedupWindow, @@ -529,9 +574,35 @@ func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails // streams created before the limits existed. See natsjs-connector.md. MaxAge: streamMaxAge(), MaxBytes: streamMaxBytes(), - }) + } + if len(bindings) > 0 { + parentStream, perr := p.ensureStream(cctx, bd, addr.GetParentAddress().GetName()) + if perr != nil { + return fmt.Errorf("ensure parent address stream: %w", perr) + } + sources, serr := p.boundSources(cctx, bd, streamName, parentStream, bindings) + if serr != nil { + return serr + } + cfg.Sources = sources + } + _, err := bd.js.CreateOrUpdateStream(cctx, cfg) return err - }) + } + var err error + if len(bindings) > 0 { + // boundSources reads the current binding set to add to it, so the + // read-modify-write has to be serialized or a concurrent subscriber + // declaring a different binding would be overwritten. The streams + // registry cannot do this job: it *collapses* concurrent callers, which + // is right when they all want the same stream but would silently drop + // one of two different binding sets. + p.bindMu.Lock() + err = build() + p.bindMu.Unlock() + } else { + err = p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), build) + } if err != nil { return "", err } @@ -539,6 +610,48 @@ func (p *natsjsProvider) ensureStream(ctx context.Context, bd *natsBrokerDetails return streamName, nil } +// boundSources merges bindings into the stream's existing source set, so a +// second address-to-address binding adds to the first rather than replacing +// it. Bindings accumulate the same way on RabbitMQ, where declaring one never +// removes another; nothing here unbinds, so a binding outlives the +// subscription that declared it, exactly as an AMQP exchange-to-exchange +// binding outlives the client that bound it. +func (p *natsjsProvider) boundSources(ctx context.Context, bd *natsBrokerDetails, + streamName, parentStream string, bindings []string) ([]*jetstream.StreamSource, error) { + sources := []*jetstream.StreamSource{} + if st, err := bd.js.Stream(ctx, streamName); err == nil { + if info, ierr := st.Info(ctx); ierr == nil { + sources = info.Config.Sources + } + } else if !errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, fmt.Errorf("read existing bindings: %w", err) + } + var from *jetstream.StreamSource + for _, s := range sources { + if s.Name == parentStream { + from = s + break + } + } + if from == nil { + from = &jetstream.StreamSource{Name: parentStream} + sources = append(sources, from) + } + for _, subject := range bindings { + if slices.ContainsFunc(from.SubjectTransforms, func(t jetstream.SubjectTransformConfig) bool { + return t.Source == subject + }) { + continue + } + // Destination == Source: the copy keeps the subject it was published + // under, which is what lets one consumer filter select both the + // messages routed in from the parent and those published here directly. + from.SubjectTransforms = append(from.SubjectTransforms, + jetstream.SubjectTransformConfig{Source: subject, Destination: subject}) + } + return sources, nil +} + func firstSubject(addr *pb.Address) string { if subs := addr.GetSubjects(); len(subs) > 0 { return subs[0] @@ -563,6 +676,17 @@ func (p *natsjsProvider) Publish(ctx context.Context, in <-chan *pb.Message, err // The server blocks on <-errChan for every message it hands the // provider (server.go: `mc <- msg` then `pubErr := <-ec`), so we // must always reply exactly once — nil on success. + if msg.GetConfirm() { + // The streaming publish is the fire-and-forget path; a client + // that wants a broker confirmation calls PublishOne, which + // returns one. amqp091 refuses the combination and so does + // this: a JetStream publish is acknowledged either way, so + // quietly accepting Confirm here would leave a client believing + // the option means the same thing on both brokers when only one + // of them honours it. + errChan <- &pb.Error{Message: unsupportedConfirmError} + continue + } errChan <- p.publishMsg(ctx, bd, msg) } } @@ -578,7 +702,7 @@ func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Er func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message) *pb.Error { addr := msg.GetAddress() - streamName, err := p.ensureStream(ctx, bd, addr.GetName()) + streamName, err := p.ensureStreamFor(ctx, bd, addr) if err != nil { return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} } @@ -589,6 +713,15 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, } var pubOpts []jetstream.PublishOpt // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). + if msg.GetPublishId() > 0 && addr.GetType() == pb.Address_QUEUE { + // JetStream would happily deduplicate this, since dedup here is a + // property of the stream rather than of the address type. Refusing it + // anyway keeps the option meaning one thing across brokers: amqp091 + // cannot offer dedup on a queue address (RabbitMQ dedup comes from + // streams), so a client allowed to set it here would be writing code + // that silently loses its deduplication on RabbitMQ. + return &pb.Error{Message: queueDedupError} + } if msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} } @@ -606,7 +739,7 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, // retry once; the first attempt was not stored (no stream answered), // so the retry cannot duplicate it. bd.knownStreams.Delete(streamName) - if _, rerr := p.ensureStream(ctx, bd, addr.GetName()); rerr == nil { + if _, rerr := p.ensureStreamFor(ctx, bd, addr); rerr == nil { _, perr = bd.js.PublishMsg(ctx, nmsg, pubOpts...) } } @@ -646,8 +779,8 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c IsFatal: true, } } - addr := source.GetAddress().GetName() - streamName, serr := p.ensureStream(ctx, bd, addr) + addr := source.GetAddress() + streamName, serr := p.ensureStreamFor(ctx, bd, addr) if serr != nil { return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", serr.Error()), IsFatal: true} } @@ -664,7 +797,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // entry and re-assert the stream instead of failing the subscribe until // the client reconnects. bd.knownStreams.Delete(streamName) - if _, rerr := p.ensureStream(ctx, bd, addr); rerr == nil { + if _, rerr := p.ensureStreamFor(ctx, bd, addr); rerr == nil { stream, serr = bd.js.Stream(tctx, streamName) } } @@ -1064,12 +1197,14 @@ func unappliedSourceOptions(source *pb.Source) []string { // messages published after the consumer is created; an absolute number -> // start at that stream sequence. // -// NOTE: numeric offsets are JetStream stream sequence numbers (as reported by -// SourceStats), which are 1-based and not portable across brokers. Sequence 0 -// therefore means "from the beginning" and maps to DeliverAll rather than an -// invalid start sequence. An unrecognized offset is an error, like -// toStreamOffset: starting a consumer at a silently different position than -// the one it asked for loses or replays data. +// NOTE: a numeric offset is an offset in the Arke/RabbitMQ-Streams sense — +// 0 names the first message in the log — not a raw JetStream sequence, so it +// round-trips through SourceStats (see offsetOf/seqOf). It still is not +// portable *across* brokers: offset 7 names the eighth message of whichever +// log is being read, and two brokers' logs hold different messages. An +// unrecognized offset is an error, like toStreamOffset: starting a consumer at +// a silently different position than the one it asked for loses or replays +// data. func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64, error) { off := source.GetOptions()["Offset"] switch strings.ToLower(off) { @@ -1080,11 +1215,11 @@ func deliverPolicyFor(source *pb.Source) (jetstream.DeliverPolicy, uint64, error case "next", "": return jetstream.DeliverNewPolicy, 0, nil default: - if seq, err := strconv.ParseUint(strings.TrimSpace(off), 10, 64); err == nil { - if seq == 0 { + if offset, err := strconv.ParseUint(strings.TrimSpace(off), 10, 64); err == nil { + if offset == 0 { return jetstream.DeliverAllPolicy, 0, nil } - return jetstream.DeliverByStartSequencePolicy, seq, nil + return jetstream.DeliverByStartSequencePolicy, seqOf(offset), nil } return jetstream.DeliverNewPolicy, 0, fmt.Errorf( "invalid offset: %q (expected first, continue, last, next, or a numeric stream sequence)", off) @@ -1166,12 +1301,42 @@ func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstrea } mu, ok := bd.activeMessages.Get(uuid) if !ok { - return nil, &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} + return nil, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} } bd.activeMessages.Delete(uuid) return mu.(inflightMsg).msg, nil } +// takeMessageForNack is takeMessage plus the message's redeliverOnNack flag, +// which only Nack needs. +func (p *natsjsProvider) takeMessageForNack(ctx context.Context, uuid string) (jetstream.Msg, bool, *pb.Error) { + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return nil, false, &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return nil, false, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} + } + bd.activeMessages.Delete(uuid) + im := mu.(inflightMsg) + return im.msg, im.redeliverOnNack, nil +} + +// markForRedelivery records that a nack of this message must put it back +// rather than terminate it. Called on every path where DeadLetter gives up +// with the message still in flight: the error it returns makes the server nack +// the same uuid, and that nack is the retry, not a rejection. +func markForRedelivery(bd *natsBrokerDetails, uuid string) { + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return + } + im := mu.(inflightMsg) + im.redeliverOnNack = true + bd.activeMessages.Add(uuid, im) +} + func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { m, perr := p.takeMessage(ctx, uuid) if perr != nil { @@ -1183,13 +1348,31 @@ func (p *natsjsProvider) Ack(ctx context.Context, uuid string) *pb.Error { return nil } -// Nack negatively acknowledges for immediate redelivery (literal AMQP nack). +// Nack rejects a message without putting it back, which is what a nack means +// on the Arke contract: amqp091 answers the same call with +// Delivery.Nack(requeue=false), so RabbitMQ drops the message (or moves it to +// the queue's dead-letter exchange, which the server instead asks for +// explicitly through DeadLetter). JetStream's Nak means the opposite — +// redeliver immediately — so naking here turns one nacked message into an +// unbounded redelivery loop between server and client, at thousands of +// deliveries a second, for as long as the subscription lives. Term is the +// primitive that carries the intended meaning: stop redelivering. +// +// The exception is the server's fallback nack after a failed DeadLetter, which +// exists to put the message back so dead-lettering can be retried. DeadLetter +// marks the message for redelivery before returning that error. func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { - m, perr := p.takeMessage(ctx, uuid) + m, redeliver, perr := p.takeMessageForNack(ctx, uuid) if perr != nil { return perr } - if err := m.Nak(); err != nil { + if redeliver { + if err := m.Nak(); err != nil { + return &pb.Error{Message: err.Error()} + } + return nil + } + if err := m.Term(); err != nil { return &pb.Error{Message: err.Error()} } return nil @@ -1233,9 +1416,15 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid } mu, ok := bd.activeMessages.Get(uuid) if !ok { - return &pb.Error{Message: fmt.Sprintf("no message with uuid %s", uuid)} + return &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} } m := mu.(inflightMsg).msg + // Every way out of this function short of a stored DLQ copy leaves the + // message in flight for the caller's fallback nack to retry, so make that + // nack mean "put it back" for as long as the attempt is unresolved. The + // success path removes the message from activeMessages outright, so the + // flag cannot outlive a dead-letter that worked. + markForRedelivery(bd, uuid) opts := source.GetOptions() dla, hasDLA := opts["DeadLetterAddress"] if !hasDLA { @@ -1364,6 +1553,25 @@ func (rt *rateTracker) observe(key string, now time.Time, count uint64) float32 return float32(float64(count-prev.count) / now.Sub(prev.when).Seconds()) } +// offsetOf converts a JetStream stream sequence to the Offset vocabulary Arke +// reports and accepts. The two do not share an origin: JetStream numbers the +// first message in a stream 1, while a RabbitMQ Stream — the vocabulary +// amqp091 established and clients read SourceStats through — numbers it 0. A +// sequence of 0 is JetStream for "no message", which has no offset, and stays +// 0 rather than going negative; callers distinguish it by LastSeq or by the +// "Offset not found" error. seqOf inverts this. +func offsetOf(seq uint64) int64 { + if seq == 0 { + return 0 + } + return int64(seq - 1) //nolint:gosec // a stream sequence cannot exceed int64 +} + +// seqOf converts an Arke Offset back to the JetStream stream sequence to start +// a consumer at. It inverts offsetOf, so an offset read from SourceStats and +// handed back as a source's Offset resumes at exactly the message it named. +func seqOf(offset uint64) uint64 { return offset + 1 } + func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb.SourceStats { stats := &pb.SourceStats{Name: source.GetName()} bd, err := p.getBrokerDetails(ctx) @@ -1385,7 +1593,7 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb now := time.Now() stats.MessageCount = int64(info.State.Msgs) //nolint:gosec stats.ConsumerCount = int32(info.State.Consumers) //nolint:gosec - stats.LastOffset = int64(info.State.LastSeq) //nolint:gosec + stats.LastOffset = offsetOf(info.State.LastSeq) durable := durableName(source) // JetStream exposes no rates, only counters; sample them between calls // (see rateTracker). The stream's LastSeq counts every publish to the @@ -1411,7 +1619,7 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb // instead of paying a second round-trip on every stats poll. ci := cons.CachedInfo() stats.MessageCount = int64(ci.NumPending) + int64(ci.NumAckPending) //nolint:gosec - stats.CurrentOffset = int64(ci.AckFloor.Stream) //nolint:gosec + stats.CurrentOffset = offsetOf(ci.AckFloor.Stream) // The stream-wide consumer count set above spans every source on // the address (streams are shared per address root), so it says // nothing about THIS source — it cannot even distinguish zero @@ -1428,5 +1636,19 @@ func (p *natsjsProvider) SourceStats(ctx context.Context, source *pb.Source) *pb stats.DeliverRate = bd.rates.observe("del/"+streamName+"/"+durable+"/"+source.GetName(), now, ci.Delivered.Consumer) } } + // A stream source reports on a log, not on a queue, and amqp091 says so in + // two ways this has to match. It fills a message count only for + // Source_QUEUE (getStreamOrQueueStats): a stream's readers each hold their + // own position in one retained log, so there is no single backlog that + // belongs to the source — the position is the reading, and it is already + // in LastOffset/CurrentOffset. And when the log has no offset to report it + // answers with RabbitMQ's own "Offset not found" (updateStatsForStream) + // rather than a silent zero. + if source.GetType() == pb.Source_STREAM { + stats.MessageCount = 0 + if info.State.LastSeq == 0 { + stats.Error = &pb.Error{Message: offsetNotFoundError} + } + } return stats } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 3622400..aa53468 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net" + "strconv" "sync" "sync/atomic" "testing" @@ -225,14 +226,18 @@ func TestRetrySynthesizesRetryCount(t *testing.T) { require.Nil(t, p.Ack(ctx, second.GetUuid())) } -func TestNackRedelivers(t *testing.T) { +// TestNackDoesNotRedeliver pins Nack to amqp091's Delivery.Nack(requeue=false): +// a nacked message is rejected, not put back. Naking it instead (JetStream's +// Nak, which means redeliver now) turns one nacked message into an unbounded +// delivery loop, because the client nacks each redelivery straight back. +func TestNackDoesNotRedeliver(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) addr := &pb.Address{Name: "events.nack", Subjects: []string{"job"}} require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) - out := make(chan *pb.Message, 2) + out := make(chan *pb.Message, 8) src := queueSource("events.nack.consumer", "events.nack", "job") subCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -241,8 +246,17 @@ func TestNackRedelivers(t *testing.T) { first := recv(t, out) require.Nil(t, p.Nack(ctx, first.GetUuid())) - second := recv(t, out) - require.Nil(t, p.Ack(ctx, second.GetUuid())) + select { + case m := <-out: + t.Fatalf("nacked message was redelivered (uuid %s); Nack must not requeue", m.GetUuid()) + case <-time.After(2 * time.Second): + } + + // Resolved server-side, not merely undelivered: nothing is left ack-pending + // to come back once AckWait expires. + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + require.EqualValues(t, 0, stats.GetMessageCount(), "nacked message still pending") } func TestDeadLetter(t *testing.T) { @@ -261,9 +275,10 @@ func TestDeadLetter(t *testing.T) { go p.Subscribe(subCtx, src, out) // Fail the first delivery, then dead-letter the redelivery, so the copy - // has a retry trail to carry. + // has a retry trail to carry. Retry (not Nack) is what asks for a + // redelivery — Nack rejects the message outright. first := recv(t, out) - require.Nil(t, p.Nack(ctx, first.GetUuid())) + require.Nil(t, p.Retry(ctx, src, first.GetUuid(), 0)) m := recv(t, out) require.Equal(t, "1", m.GetHeaders()[retryCountHeaderName], "redelivery precondition") require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) @@ -842,7 +857,9 @@ func TestAckUnknownUUID(t *testing.T) { perr := p.Ack(ctx, "does-not-exist") require.NotNil(t, perr) - assert.Contains(t, perr.GetMessage(), "no message with uuid") + // Worded exactly as amqp091 words it: a client matching on the text must + // not have to know which connector answered. + assert.Equal(t, "No message with uuid does-not-exist", perr.GetMessage()) } func TestStreamConfigFromEnv(t *testing.T) { @@ -1006,7 +1023,9 @@ func TestDeliverPolicyFor(t *testing.T) { check("last", jetstream.DeliverLastPolicy, 0) check("next", jetstream.DeliverNewPolicy, 0) check("", jetstream.DeliverNewPolicy, 0) - check("100", jetstream.DeliverByStartSequencePolicy, 100) + // A numeric offset counts from 0 like a RabbitMQ Stream's, so it is one + // less than the JetStream sequence it starts the consumer at. + check("100", jetstream.DeliverByStartSequencePolicy, 101) check("0", jetstream.DeliverAllPolicy, 0) // offset 0 == from the beginning pol, seq, err := deliverPolicyFor(&pb.Source{}) @@ -1406,6 +1425,66 @@ func TestStreamOffsetReplay(t *testing.T) { }) } +// TestStreamOffsetContinueResumes: offset "continue" resumes a group-less +// stream source where its last subscription stopped, instead of replaying the +// log from the start like "first". amqp091 answers "continue" from RabbitMQ +// Streams' server-side offset tracking (QueryOffset by consumer name, +1); +// here the source's durable holds the position, so a re-subscribe under the +// same name picks up after the messages it already acked. +func TestStreamOffsetContinueResumes(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.cont", Subjects: []string{"e"}} + publish := func(t *testing.T, n int, tag string) { + t.Helper() + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, + &pb.Message{Address: addr, Body: []byte(fmt.Sprintf("%s%d", tag, i))})) + } + } + src := func() *pb.Source { + return &pb.Source{ + Name: "events.cont.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.cont", Subjects: []string{"e"}}, + Options: map[string]string{"Offset": "continue"}, + } + } + + // With nothing read yet there is no stored position, so "continue" starts + // at the beginning — RabbitMQ's QueryOffset finds no offset and falls back + // to offset 0 the same way. + publish(t, 2, "first") + drain := func(t *testing.T, want int) []string { + t.Helper() + out := make(chan *pb.Message, want+4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src(), out) + got := make([]string, 0, want) + for i := 0; i < want; i++ { + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + got = append(got, string(m.GetBody())) + } + // Nothing beyond what was asked for: a replay would show up here. + select { + case m := <-out: + t.Fatalf("unexpected extra message %q; continue replayed the log", m.GetBody()) + case <-time.After(time.Second): + } + return got + } + require.Equal(t, []string{"first0", "first1"}, drain(t, 2)) + + // The second subscription must see only what was published after the + // first one stopped. + publish(t, 3, "second") + require.Equal(t, []string{"second0", "second1", "second2"}, drain(t, 3), + "continue replayed already-acked messages instead of resuming") +} + // TestStreamOffsetNextSkipsBacklog: offset "next" (and the default) positions a // new consumer at the tail — it sees only messages published after it was // created, matching RabbitMQ Streams "next". @@ -1467,12 +1546,13 @@ func TestStreamOffsetLastAndNumeric(t *testing.T) { } }) - t.Run("numeric_offset_starts_at_sequence", func(t *testing.T) { + t.Run("numeric_offset_starts_at_offset", func(t *testing.T) { out := make(chan *pb.Message, n) subCtx, cancel := context.WithCancel(ctx) defer cancel() - // JetStream sequence 3 == m2 (1-based); expect m2 then m3. - go p.Subscribe(subCtx, streamSource("events.offset.num", "events.offset", "3", "e"), out) + // Offsets count from 0 like a RabbitMQ Stream's, so offset 2 is the + // third message, m2 — not JetStream's sequence 2, which is m1. + go p.Subscribe(subCtx, streamSource("events.offset.num", "events.offset", "2", "e"), out) got := []string{} for i := 0; i < 2; i++ { @@ -1480,7 +1560,27 @@ func TestStreamOffsetLastAndNumeric(t *testing.T) { got = append(got, string(m.GetBody())) require.Nil(t, p.Ack(ctx, m.GetUuid())) } - assert.Equal(t, []string{"m2", "m3"}, got, "numeric offset starts at the given stream sequence") + assert.Equal(t, []string{"m2", "m3"}, got, "numeric offset starts at the named message") + }) + + // The offset vocabulary has to round-trip: an offset read from SourceStats + // and handed back as a source's Offset must name the same message. A + // connector reporting JetStream's 1-based sequence but accepting it back + // as a 0-based offset would quietly skip one message per hop. + t.Run("offset_from_stats_round_trips", func(t *testing.T) { + stats := p.SourceStats(ctx, streamSource("events.offset.rt", "events.offset", "first", "e")) + require.Nil(t, stats.GetError()) + require.EqualValues(t, n-1, stats.GetLastOffset(), "last offset names the final message") + + out := make(chan *pb.Message, n) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + last := strconv.FormatInt(stats.GetLastOffset(), 10) + go p.Subscribe(subCtx, streamSource("events.offset.rt2", "events.offset", last, "e"), out) + + m := recv(t, out) + assert.Equal(t, "m3", string(m.GetBody()), "LastOffset fed back must start at the last message") + require.Nil(t, p.Ack(ctx, m.GetUuid())) }) } @@ -1608,9 +1708,154 @@ func TestDurableConflictBeyondStartPositionStaysFatal(t *testing.T) { assert.Contains(t, perr.GetMessage(), "create consumer") } +// TestPublishContractRefusals pins the two publish combinations amqp091 +// refuses, so a client cannot write code against natsjs that silently means +// something else on RabbitMQ. Both error strings are amqp091's, verbatim. +func TestPublishContractRefusals(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Deduplication on a queue address: RabbitMQ gets dedup from streams, so + // amqp091 cannot offer it on a queue at all. + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.q", Type: pb.Address_QUEUE, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")}) + require.NotNil(t, perr) + assert.Equal(t, queueDedupError, perr.GetMessage()) + + // ...but the same publish to a non-queue address is fine. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.t", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")})) + + // Confirm on the fire-and-forget streaming publish. + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 2) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + in <- &pb.Message{Address: &pb.Address{Name: "events.t", Subjects: []string{"k"}}, + Body: []byte("x"), Confirm: true} + select { + case e := <-errChan: + require.NotNil(t, e) + assert.Equal(t, unsupportedConfirmError, e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish error") + } + // Exactly one reply per message: the server does an unconditional receive + // after each send, so a second value here would answer the *next* message. + select { + case e := <-errChan: + t.Fatalf("second reply for one message: %v", e) + case <-time.After(time.Second): + } +} + +// TestParentAddressBindingRoutes covers address-to-address binding: a child +// address bound to a parent receives what is published to the parent under the +// bound keys, still receives what is published to it directly, and does not +// receive what the parent routes elsewhere. +func TestParentAddressBindingRoutes(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.child", + Type: pb.Address_FILTER, + Subjects: []string{"bound.one", "bound.two"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 8) + src := &pb.Source{Name: "events.child.consumer", Type: pb.Source_QUEUE, Address: child, + Options: map[string]string{"Offset": "first"}} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + time.Sleep(500 * time.Millisecond) + + pub := func(t *testing.T, addrName, key, body string) { + t.Helper() + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: addrName, Subjects: []string{key}}, + Body: []byte(body)})) + } + // Routed in from the parent under a bound key... + pub(t, "events.parent", "bound.one", "routed") + // ...not routed: the parent has no binding to the child for this key. + pub(t, "events.parent", "unbound.key", "not-routed") + // ...and published straight to the child. + pub(t, "events.child", "bound.two", "direct") + + got := map[string]bool{} + deadline := time.After(6 * time.Second) + for len(got) < 2 { + select { + case m := <-out: + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + case <-deadline: + t.Fatalf("timed out; received %v", got) + } + } + assert.True(t, got["routed"], "a message published to the parent under a bound key must reach the child") + assert.True(t, got["direct"], "a direct publish to the child must still arrive") + + select { + case m := <-out: + t.Fatalf("child received %q, which the parent does not route to it", m.GetBody()) + case <-time.After(2 * time.Second): + } +} + +// TestParentAddressBindingsAccumulate: a second address-to-address binding on +// the same address adds to the first rather than replacing it, the way +// declaring a binding on RabbitMQ never removes another. +func TestParentAddressBindingsAccumulate(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.acc.parent", Type: pb.Address_TOPIC} + childWith := func(subjects ...string) *pb.Address { + return &pb.Address{Name: "events.acc.child", Type: pb.Address_TOPIC, + Subjects: subjects, ParentAddress: parent} + } + sub := func(t *testing.T, name string, addr *pb.Address) chan *pb.Message { + t.Helper() + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + go p.Subscribe(subCtx, &pb.Source{Name: name, Type: pb.Source_QUEUE, Address: addr, + Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + return out + } + + first := sub(t, "events.acc.first", childWith("key.one")) + // The second subscriber declares a different binding on the same address. + second := sub(t, "events.acc.second", childWith("key.two")) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.acc.parent", Subjects: []string{"key.one"}}, + Body: []byte("one")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.acc.parent", Subjects: []string{"key.two"}}, + Body: []byte("two")})) + + // The first subscriber's binding must have survived the second's declare. + m := recv(t, first) + assert.Equal(t, "one", string(m.GetBody()), "the earlier binding was dropped by a later one") + m = recv(t, second) + assert.Equal(t, "two", string(m.GetBody())) +} + // TestEmptyRoutingKeyDelivers covers fanout-style publishes that carry no -// routing key: they map to the bare ".~" subject, which both the stream -// and a pattern-less consumer capture. +// routing key: they map to the bare ".~" subject, which the stream +// captures and a consumer selects by binding the empty routing key. The +// binding has to be declared — a source with no bindings receives nothing, +// whatever the routing key (see TestNoBindingsReceiveNothing). func TestEmptyRoutingKeyDelivers(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -1621,13 +1866,45 @@ func TestEmptyRoutingKeyDelivers(t *testing.T) { out := make(chan *pb.Message, 1) subCtx, cancel := context.WithCancel(ctx) defer cancel() - go p.Subscribe(subCtx, queueSource("events.fanout.consumer", "events.fanout"), out) + go p.Subscribe(subCtx, queueSource("events.fanout.consumer", "events.fanout", ""), out) m := recv(t, out) assert.Equal(t, "no-key", string(m.GetBody())) require.Nil(t, p.Ack(ctx, m.GetUuid())) } +// TestNoBindingsReceiveNothing pins the AMQP rule that a source with no +// routing-key bindings receives nothing. amqp091 declares no binding at all +// for such a source (declareBinding iterates an empty subject list), so +// selecting the whole address instead — which a pattern-less consumer used to +// do — hands the source every message published to it. +func TestNoBindingsReceiveNothing(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 4) + src := queueSource("events.unbound.consumer", "events.unbound") + src.Address.Subjects = nil + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + // Let the consumer establish before publishing, so a delivery would race + // in rather than be missed. + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.unbound"}, Body: []byte("no-key")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.unbound", Subjects: []string{"some.key"}}, + Body: []byte("keyed")})) + + select { + case m := <-out: + t.Fatalf("source with no bindings received %q", m.GetBody()) + case <-time.After(2 * time.Second): + } +} + func TestDirectAddressBindingIsExact(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -2127,6 +2404,42 @@ func TestRateTracker(t *testing.T) { assert.Zero(t, rt.observe("other", t0, 1)) } +// TestSourceStatsStreamSourceReportsOffsetsNotDepth pins the two ways a +// Source_STREAM's stats differ from a queue's on amqp091: no message count +// (the readers of a shared log have no common backlog — the offsets are the +// reading), and RabbitMQ's "Offset not found" when the log holds no offset +// yet, rather than a silent zero. +func TestSourceStatsStreamSourceReportsOffsetsNotDepth(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + src := &pb.Source{ + Name: "events.log.consumer", + Type: pb.Source_STREAM, + Address: &pb.Address{Name: "events.log", Type: pb.Address_STREAM, Subjects: []string{"e"}}, + } + + // An empty log has no offset to report. + bd, bderr := p.getBrokerDetails(ctx) + require.NoError(t, bderr) + _, serr := p.ensureStream(ctx, bd, "events.log") + require.Nil(t, serr) + empty := p.SourceStats(ctx, src) + require.NotNil(t, empty.GetError()) + assert.Equal(t, offsetNotFoundError, empty.GetError().GetMessage()) + + addr := &pb.Address{Name: "events.log", Subjects: []string{"e"}} + const n = 5 + for i := 0; i < n; i++ { + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + } + + stats := p.SourceStats(ctx, src) + require.Nil(t, stats.GetError()) + assert.EqualValues(t, n-1, stats.GetLastOffset(), "offsets count from 0, so the fifth message is offset 4") + assert.Zero(t, stats.GetMessageCount(), "a stream source reports no backlog, like amqp091") +} + // TestSourceStatsRates verifies publish/deliver rates are sampled between // SourceStats calls: the first call baselines at zero, and a later call after // publishing and consuming reports positive rates. From b4a6497167774313d772f90d50fe30816bb7ce2f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 15 Jul 2026 12:09:13 -0400 Subject: [PATCH 50/77] docs: record why storage sits at the address, not the source The address-to-stream, source-to-consumer mapping puts message storage one level above where AMQP puts it, which is why per-source MessageTTL and max-length are not honored and acked messages are not deleted on ack. The obvious alternative -- a stream per source, sourcing from a per-address origin -- is more attractive in outline than in fact, and had been re-litigated from first principles more than once. Record the decision and the evidence against the alternative: it does not deliver expire-into-the-dead-letter-exchange, which JetStream cannot express at any topology (no advisory fires on limit eviction, and subject delete markers are a key-value feature -- no body, last message on a subject only, and stamped with a rollup that purges the subject); it weakens publish confirmation to the origin stream; it costs a stored copy, a replication group and a synchronous write per message per source; WorkQueuePolicy is narrower than a queue (errs 10099/10100/10101); and origin retention becomes a silent-loss dependency. Also record why InterestPolicy is not a cheaper route to delete-on-ack, and the conditions under which to revisit. Record a fourth accepted consequence: a transient source becomes an ephemeral consumer, and no start position for it is faithful. A queue's existence defines what is retained for that queue, while a stream retains what the address received regardless of listeners, so DeliverNew skips messages a queue that ought to have existed would hold, and DeliverAll replays history a fresh transient queue never had. DeliverNew is the narrower error, but it is a trade rather than a mapping. Also rewrap an over-length line left behind in the source-stats limitation. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 89 +++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index a8dcad1..534dbb9 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -360,6 +360,92 @@ would silently delete a down consumer's backlog and regress the durable-queue behavior. The default therefore exceeds any realistic outage, and `MaxBytes` is intended as the primary disk guard. +## Design decision: storage at the address, not the source + +The connector maps an address to a stream and a source to a consumer, so +message storage sits one level higher than AMQP puts it. In AMQP an exchange +is a stateless routing function and the queue owns storage *and* per-queue +policy — `MessageTTL`, max-length, dead-lettering, delete-on-ack. In +JetStream the stream owns storage and policy, and a consumer is only a +cursor over it. Mapping address to stream therefore trades per-source policy +away, and this section records why that trade is deliberate, because the +obvious alternative is more attractive in outline than in fact. + +The consequences accepted are: per-source `MessageTTL` and max-length cannot +be honored (a shared stream cannot carry one source's duration without +flapping it for every other source on the same address); acked messages stay +in the log until a limit evicts them rather than being deleted on ack; and +publish-rate statistics are per address rather than per source. + +A fourth consequence is subtler and has no clean answer: a transient source +becomes an ephemeral consumer, and no start position for it is faithful. In +AMQP the existence of a queue defines what is retained *for that queue* — a +transient queue holds nothing from before it was declared, and is deleted +when its last consumer leaves. A stream instead retains everything the +address received, regardless of who is listening. `DeliverNew` is therefore +wrong whenever a transient source stands in for a queue that ought to have +existed already and accumulated: a dead-letter target consumed only after +the fact is the clearest case, where the messages are in the stream and the +ephemeral consumer starts past them. `DeliverAll` is wrong in the opposite +direction, replaying history a freshly declared transient queue could never +have held, bounded only by `MaxAge`. The connector picks `DeliverNew` +because replaying a busy address's whole retained log into a temporary +consumer is the more damaging error, but this is a trade rather than a +mapping. A source that must not miss earlier messages should be durable — +non-auto-delete, or carrying a consumer group — which is the AMQP-faithful +way to say the queue exists independently of its consumers. + +The alternative is a stream per source. Because two streams may not listen +on overlapping subjects, each source stream cannot simply capture its bound +subjects — sources binding `orders.*` and `orders.created` would collide. +It would instead have to *source* from a per-address origin stream with a +subject filter, the same mechanism address-to-address binding uses above. +That buys per-source `MaxAge` and max-length, delete-on-ack via +`WorkQueuePolicy`, and per-source statistics. It was rejected for five +reasons, the first of which is decisive: + +- **It does not buy the feature that motivates it.** The usual reason to + want per-source TTL is AMQP's expire-into-the-dead-letter-exchange idiom, + and JetStream cannot express it at any topology. There is no advisory for + expiry: the advisory set covers consumer max-deliveries, nak, term, + lifecycle and leader elections, but nothing fires when a limit evicts a + message. Subject delete markers are not a substitute — they are a + key-value feature, carrying no message body, emitted only for the last + message on a subject, and stamped `Nats-Rollup: sub`, which purges the + subject. Per-source streams would give TTL *deletion*, never TTL *routing*. +- **Publish confirmation would weaken.** Sourcing is asynchronous, so a + confirmed publish would mean "durable in the origin stream", not "durable + in the bound source" — a real regression against AMQP, where a publisher + confirm covers every bound queue. +- **It multiplies cost per message and per source.** Every message is stored + once more (origin plus each source), every source adds a replication group, + and on a replicated stream with synchronous flushing the sourcing hop is an + additional synchronous write per message per source, on the path that + already bounds throughput. +- **`WorkQueuePolicy` is narrower than a queue.** It rejects multiple + unfiltered consumers (err 10099), non-unique filtered consumers (10100), + and any consumer that is not deliver-all (10101), so the delete-on-ack prize + comes with constraints a queue does not have. +- **Origin retention becomes a correctness dependency.** If the origin's + `MaxAge` elapses before a lagging source stream has sourced a message, the + message is lost silently — a failure mode the current topology does not have. + +Switching the shared stream to `InterestPolicy` was also considered, as a +cheaper route to delete-on-ack that keeps one stream per address. It is +wrong here: interest is evaluated at publish time, so a message published +before any consumer exists is discarded immediately. That would break +`Offset: first` replay and any stream-typed source, which exist precisely to +read a retained log. Queue-like and log-like sources share an address +stream and want opposite retention; `LimitsPolicy` is the only policy correct +for both, and over-retention is a storage cost rather than a correctness one. + +Revisit this decision if a deployment needs per-source expiry or max-length, +or if delete-on-ack becomes a storage problem that `MaxAge` and `MaxBytes` +cannot contain — and only if the weaker publish-confirm guarantee is +acceptable there. Note that `Retention` cannot be changed into or out of +`WorkQueuePolicy` on a live stream, so any such move is a stream-recreate +migration, not a configuration change. + ## Operational resilience Five mechanisms harden the connector against a cold or busy broker. All are @@ -515,7 +601,8 @@ amqp091 connector, so existing client sources validate unchanged: view. A `STREAM` source reports no message count at all, as on amqp091: the readers of one retained log have no common backlog, and their reading is the offset pair instead. A stream with no offset to report answers with - RabbitMQ's own `Offset not found` error rather than a silent zero. A durable source's consumer count is likewise per source: the number + RabbitMQ's own `Offset not found` error rather than a silent zero. A + durable source's consumer count is likewise per source: the number of clients with an open pull request on its consumer — a client working through a full pull buffer can briefly read as zero — while sources without a durable consumer report the stream-wide consumer count, which From 6d40a5ea6fe3f1b5de3cdb875f5c94378a9cb468 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Thu, 16 Jul 2026 13:38:40 -0400 Subject: [PATCH 51/77] fix: apply the Expires option as the consumer's InactiveThreshold Expires (AMQP x-expires: delete the queue after this many milliseconds without consumers) was accepted, warned about, and silently ignored - filed alongside MessageTTL as blocked by the shared per-address stream. That classification was wrong: MessageTTL governs message retention and is genuinely stream-wide here, but Expires governs the source's lifetime, and consumers are per-source, so it maps cleanly onto the consumer's InactiveThreshold - which the connector already set, just hardcoded to 5 minutes for ephemerals and never for durables. A client overriding Expires was honored by amqp091 and silently diverged here. Now the value is parsed and applied to both kinds of consumer (the amqp091 connector likewise sets x-expires on any queue; nats-server supports the threshold on durables since 2.9 and allows updating it). Unset keeps the previous defaults: transients expire after 5 minutes - the same default amqp091 applies to auto-delete/exclusive queues without x-expires - and durables never expire. A non-integer value is rejected with amqp091's exact error string; a non-positive value, which the AMQP broker itself refuses, is rejected too rather than silently meaning never (durable) or the server default (ephemeral). Only MessageTTL remains in the accepted-but-unapplied warn. The design doc's parity table, options table, and Known limitations no longer lump the two options together. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 35 +++--- internal/provider/connectors/natsjs/natsjs.go | 59 +++++++++- .../provider/connectors/natsjs/natsjs_test.go | 111 +++++++++++++++++- 3 files changed, 183 insertions(+), 22 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 534dbb9..c273636 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -347,8 +347,9 @@ an acked message stays in the log and is only evicted when a configured limit To stop a stream growing without bound, `ensureStream` sets: - `MaxAge` (default 72h, override with `NATSJS_STREAM_MAX_AGE`) — the time - guard. It is the natural map for AMQP `MessageTTL` / `Expires` (both - durations). It is mutable, so it also reins in streams created before a limit + guard. It is the natural map for AMQP `MessageTTL` (a message-age duration; + `Expires` is queue disuse and maps to the consumer instead — see Source + options). It is mutable, so it also reins in streams created before a limit existed. - `MaxBytes` (default unlimited, override with `NATSJS_STREAM_MAX_BYTES`) — an optional hard storage cap. With `discard=old` it only evicts near the cap, so @@ -533,7 +534,8 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Dead-letter (DLX) | Proxy | No native DLX; republish to DLQ subject then `Term()`. | | Header-filter exchange (`Filter` / `Match`) | Proxy | NATS routes on subject; evaluated in `evaluateFilters` (see limitations). | | Address-to-address binding (`ParentAddress`) | Native | The child's stream sources the bound subjects from the parent's, keeping each message's subject. | -| `MessageTTL` / `Expires` | Partial | Mapped onto stream-level `MaxAge` (see limitations). | +| `MessageTTL` (per-queue message TTL) | Drop | Accepted but not applied; retention is the stream-level `MaxAge` (see limitations). | +| `Expires` (queue disuse expiry) | Native | Consumer `InactiveThreshold`; the consumer is deleted after that long without an attached client. | | Dead-letter on message expiry | Drop | RabbitMQ expires a message off a queue into its DLX; per-source TTL is not applied here, so nothing expires per source to dead-letter (see limitations). | | Source stats (depth / consumers) | Native | JetStream stream / consumer `Info`. | | Max message size | Partial | NATS caps a payload at the server's `max_payload` (1MB default) where RabbitMQ's `max_message_size` default is 16MB (see limitations). | @@ -549,7 +551,7 @@ amqp091 connector, so existing client sources validate unchanged: | Key | Type | Description | | --- | --- | --- | | `MessageTTL` | string (ms) | Accepted for compatibility; not applied per source — retention is the stream-wide `NATSJS_STREAM_MAX_AGE` (see Known limitations). A warning is logged at subscribe time. | -| `Expires` | string (ms) | Accepted for compatibility; not applied — streams are shared per address root and are not deleted when a source goes unused. A warning is logged at subscribe time. | +| `Expires` | string (ms) | How long the source may go without an attached consumer before the broker deletes it (AMQP `x-expires`), mapped to the consumer's `InactiveThreshold`. Unset keeps the defaults: transient sources expire after 5 minutes (the same default amqp091 applies to auto-delete/exclusive queues), durable sources never expire. Deletion removes the consumer and its ack state only; the messages stay in the shared stream under its own retention. A non-integer or non-positive value fails the subscribe. | | `DeadLetterAddress` | string | Address whose stream receives dead-lettered messages. | | `DeadLetterSubject` | string | Routing-key override for dead-lettered messages; when unset, the copy keeps the message's original routing key (RabbitMQ's default dead-letter behavior). | | `Offset` | string | Stream starting offset (`first`, `continue`, `last`, `next`, or a number counting from 0, as reported by `SourceStats`). | @@ -558,17 +560,20 @@ amqp091 connector, so existing client sources validate unchanged: ## Known limitations - **Per-source TTL fidelity, and expiry-driven dead-lettering.** The - connector uses one stream per address root, so a per-source `MessageTTL` / - `Expires` cannot be mapped onto the shared stream without one source's - value flapping another's retention. Both options are therefore accepted but - not applied, and a subscribe that sets either logs a warning naming the - source — silent divergence in data retention is the one place a client must - not have to read a design doc to notice. Retention comes from the - stream-wide `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` - configuration. True per-source TTL (or switching queue sources to a - delete-on-ack policy such as `WorkQueuePolicy` / `InterestPolicy`) needs a - per-source stream topology, and `Retention` is immutable, so that is a - stream-recreate migration rather than an in-place change. + connector uses one stream per address root, so a per-source `MessageTTL` + cannot be mapped onto the shared stream without one source's value + flapping another's retention. That option is therefore accepted but not + applied, and a subscribe that sets it logs a warning naming the source — + silent divergence in data retention is the one place a client must not + have to read a design doc to notice. (`Expires` is different: it governs + the *source's* lifetime, not its messages', and consumers are per-source, + so it maps cleanly onto the consumer's `InactiveThreshold` and IS + applied — see Source options.) Retention comes from the stream-wide + `NATSJS_STREAM_MAX_AGE` / `NATSJS_STREAM_MAX_BYTES` configuration. True + per-source TTL (or switching queue sources to a delete-on-ack policy such + as `WorkQueuePolicy` / `InterestPolicy`) needs a per-source stream + topology, and `Retention` is immutable, so that is a stream-recreate + migration rather than an in-place change. A consequence worth stating on its own: RabbitMQ's per-queue TTL doubles as a routing mechanism, expiring a message off its queue and *into* the diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 33a050b..f16b329 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -759,15 +759,25 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if err != nil { return &pb.Error{Message: err.Error(), IsFatal: true} } - // MessageTTL/Expires are accepted so existing client sources validate - // against SupportedSourceOptions, but retention here is stream-wide: warn - // instead of silently ignoring them, so a source owner asking for a short + // MessageTTL is accepted so existing client sources validate against + // SupportedSourceOptions, but retention here is stream-wide: warn + // instead of silently ignoring it, so a source owner asking for a short // TTL learns their data outlives it without reading the design doc. if unapplied := unappliedSourceOptions(source); len(unapplied) > 0 { util.Logger.Warn( "natsjs: source {0} sets {1}, which natsjs accepts but does not apply; retention is stream-wide (NATSJS_STREAM_MAX_AGE / NATSJS_STREAM_MAX_BYTES)", source.GetName(), strings.Join(unapplied, ", ")) } + // Expires, by contrast, IS applied: it becomes the consumer's + // InactiveThreshold (see below), so validate it up front before any + // topology is created. + expires, xerr := expiresThreshold(source) + if xerr != nil { + return &pb.Error{ + Message: fmt.Sprintf("source %q: %s", source.GetName(), xerr.Error()), + IsFatal: true, + } + } // A single-active stream source must name the ConsumerGroup its instances // coordinate through (see durableName); amqp091 rejects this combination // with the same message. Without a group each subscriber would get its own @@ -836,6 +846,17 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // DeliverPolicy only applies on first creation; reconnects resume from the // durable's stored ack position. Transient (auto-delete/exclusive/TEMPORARY) // sources stay ephemeral and auto-expire after InactiveThreshold. + // + // The Expires option (AMQP x-expires: delete the queue after this many ms + // without consumers) maps onto InactiveThreshold for BOTH kinds — the + // broker deletes the consumer once nothing has pulled from it for the + // threshold, exactly the disuse-deletion x-expires asks for. Only the + // consumer is deleted; the messages stay in the shared per-address stream + // under its own retention (see Known limitations in the design doc). + // Unset keeps the amqp091 defaults: transient sources expire after + // defaultInactiveThreshold (amqp091 defaults auto-delete/exclusive queues + // to the same 5 minutes), durables never expire. + consCfg.InactiveThreshold = expires if durable := durableName(source); durable != "" { consCfg.Durable = durable if source.GetSingleActiveConsumer() { @@ -851,7 +872,7 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c consCfg.PriorityGroups = []string{sacPriorityGroup} consCfg.PinnedTTL = sacPinnedTTL() } - } else { + } else if expires == 0 { consCfg.InactiveThreshold = defaultInactiveThreshold } @@ -1175,10 +1196,11 @@ func deleteEphemeralConsumer(ctx context.Context, stream jetstream.Stream, name // unappliedSourceOptions returns the retention options the source sets that // natsjs accepts (so sources written for the amqp091 connector still validate) // but does not apply — retention on natsjs is stream-wide (see streamMaxAge -// and the design doc's Known limitations). +// and the design doc's Known limitations). Expires is NOT in this list: it +// maps onto the consumer's InactiveThreshold (see expiresThreshold). func unappliedSourceOptions(source *pb.Source) []string { var unapplied []string - for _, opt := range []string{"MessageTTL", "Expires"} { + for _, opt := range []string{"MessageTTL"} { if source.GetOptions()[opt] != "" { unapplied = append(unapplied, opt) } @@ -1186,6 +1208,31 @@ func unappliedSourceOptions(source *pb.Source) []string { return unapplied } +// expiresThreshold parses the Expires source option — AMQP x-expires, the +// number of milliseconds a queue may go unused before the broker deletes it — +// into the consumer InactiveThreshold that implements the same disuse +// deletion here. Zero (with nil error) means the option is unset and the +// caller applies its kind's default. The error strings mirror the amqp091 +// connector, which validates the same option the same way; the broker itself +// rejects a non-positive x-expires, so that case is refused here too rather +// than passed through (a zero threshold would mean "never expire" for a +// durable but "server default" for an ephemeral — neither is what the client +// asked for). +func expiresThreshold(source *pb.Source) (time.Duration, error) { + raw := source.GetOptions()["Expires"] + if raw == "" { + return 0, nil + } + val, err := strconv.Atoi(raw) + if err != nil { + return 0, errors.New("value for Expires option must be a quoted integer") + } + if val <= 0 { + return 0, errors.New("value for Expires option must be a positive integer") + } + return time.Duration(val) * time.Millisecond, nil +} + // deliverPolicyFor maps the RabbitMQ-Streams `Offset` option onto a JetStream // DeliverPolicy, mirroring the amqp091 connector's toStreamOffset so both // connectors accept the same offset vocabulary. The second return value is the diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index aa53468..24d8b91 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -654,6 +654,93 @@ func TestPrefetchMapsToMaxAckPending(t *testing.T) { assert.Equal(t, -1, maxAckPending("events.prefetch.unlimited", 0)) } +// TestExpiresSetsInactiveThreshold: the Expires option (AMQP x-expires — +// delete the queue after this many ms without consumers) must become the +// consumer's InactiveThreshold, for durables and ephemerals alike; unset +// keeps the defaults (durables never expire, transients get +// defaultInactiveThreshold). Pre-fix the value was accepted, warned about, +// and silently ignored: durables got no threshold and ephemerals always got +// the 5-minute default, so a client shortening or lengthening its queue's +// disuse lifetime diverged from amqp091 without any signal. +func TestExpiresSetsInactiveThreshold(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + durableThreshold := func(name string, opts map[string]string) time.Duration { + t.Helper() + src := queueSource(name, "events.expires", "e") + for k, v := range opts { + src.Options[k] = v + } + src.DeclareOnly = true + require.Nil(t, p.Subscribe(ctx, src, nil)) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.expires")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(src)) + require.NoError(t, cerr) + return cons.CachedInfo().Config.InactiveThreshold + } + + assert.Equal(t, 2*time.Minute, + durableThreshold("events.expires.set", map[string]string{"Expires": "120000"}), + "a durable's Expires must become its InactiveThreshold") + assert.Zero(t, durableThreshold("events.expires.unset", nil), + "a durable without Expires never expires") + + // Ephemeral consumers are deleted eagerly when their subscription ends, + // so inspect the config while the subscription is live. + ephemeralThreshold := func(name string, opts map[string]string) time.Duration { + t.Helper() + src := &pb.Source{ + Name: name, + Type: pb.Source_QUEUE, + AutoDelete: true, // transient -> ephemeral consumer + Address: &pb.Address{Name: "events.expires.tmp." + name, Subjects: []string{"e"}}, + Options: opts, + } + require.Empty(t, durableName(src), "test source must map to an ephemeral consumer") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) + var threshold time.Duration + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor(src.GetAddress().GetName())) + if serr != nil { + return false + } + infos := stream.ListConsumers(ctx) + for info := range infos.Info() { + threshold = info.Config.InactiveThreshold + return true + } + return false + }, 10*time.Second, 20*time.Millisecond, "ephemeral consumer never appeared") + return threshold + } + + assert.Equal(t, time.Minute, + ephemeralThreshold("eph.set", map[string]string{"Expires": "60000"}), + "a transient source's Expires must become its InactiveThreshold") + assert.Equal(t, defaultInactiveThreshold, ephemeralThreshold("eph.unset", nil), + "a transient source without Expires keeps the default threshold") +} + +func TestSubscribeInvalidExpiresRejected(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + for _, bad := range []string{"10m", "0", "-5"} { + src := queueSource("events.expires.bad", "events.expires.bad", "e") + src.Options["Expires"] = bad + src.DeclareOnly = true // the reject must come before any topology work + serr := p.Subscribe(ctx, src, nil) + require.NotNil(t, serr, "Expires=%q must be rejected", bad) + assert.True(t, serr.GetIsFatal()) + assert.Contains(t, serr.GetMessage(), "value for Expires option must be") + } +} + func TestDeduplication(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -2553,6 +2640,28 @@ func TestUnappliedSourceOptions(t *testing.T) { assert.Empty(t, unappliedSourceOptions(src(nil))) assert.Empty(t, unappliedSourceOptions(src(map[string]string{"Offset": "first", "MessageTTL": ""}))) assert.Equal(t, []string{"MessageTTL"}, unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000"}))) - assert.Equal(t, []string{"MessageTTL", "Expires"}, + // Expires is applied (consumer InactiveThreshold), so it must not be + // reported — and warned about — as unapplied. + assert.Equal(t, []string{"MessageTTL"}, unappliedSourceOptions(src(map[string]string{"MessageTTL": "300000", "Expires": "600000"}))) } + +func TestExpiresThreshold(t *testing.T) { + src := func(opts map[string]string) *pb.Source { return &pb.Source{Options: opts} } + + d, err := expiresThreshold(src(nil)) + require.NoError(t, err) + assert.Zero(t, d, "unset means the kind's default applies") + + d, err = expiresThreshold(src(map[string]string{"Expires": "600000"})) + require.NoError(t, err) + assert.Equal(t, 10*time.Minute, d) + + _, err = expiresThreshold(src(map[string]string{"Expires": "10m"})) + require.EqualError(t, err, "value for Expires option must be a quoted integer", + "error string mirrors the amqp091 connector") + _, err = expiresThreshold(src(map[string]string{"Expires": "0"})) + require.EqualError(t, err, "value for Expires option must be a positive integer") + _, err = expiresThreshold(src(map[string]string{"Expires": "-5"})) + require.EqualError(t, err, "value for Expires option must be a positive integer") +} From 2c1e69e0861abc89d815ae541e3a8564b8d7c70e Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Thu, 16 Jul 2026 14:04:02 -0400 Subject: [PATCH 52/77] fix: keep the consume stream open after a client-initiated Disconnect A Disconnect with a live subscription ended that Subscribe, and the server ends the caller's whole consume stream when Subscribe returns - the client saw end-of-stream. The amqp091 connector leaves that stream open: closing the AMQP connection leaves its subscribe loop blocked in a select none of whose channels can fire again (the close notification arrives once as a zero-value error while the connection state still reads connected, and the shim's delivery-forwarding goroutine exits without closing its output channel), so the loop just goes quiet and the integration suite pins the observable result: a client that disconnects and then acks its in-flight messages gets each ack answered with a per-ack 'could not retrieve broker details' failure rather than end-of-stream (TestSubscribeAckInvalidIDNoConnect). Subscribe now parks until the stream's own context ends when the subscription was torn down by this client's Disconnect. Delivery cannot resume either way - the disconnect already stopped the consume machinery and drained the connection. The case is discriminated by a clientDisconnect flag (mirroring amqp091's) set before the consume contexts are stopped, not by the CLOSED connection state alone, so a connection the NATS library closes terminally still ends the subscription as before. Also aligns the no-broker-details error string with amqp091 ('for this connection'), which that test asserts as a prefix. Integration standing moves to 42 passing (was 41); the remaining 8 failures are the known, triaged set. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 10 +++++ internal/provider/connectors/natsjs/natsjs.go | 32 +++++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 43 +++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index c273636..3188ab1 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -498,6 +498,16 @@ forming a raft group per stream and per durable consumer: and, after a storage wipe, the stream. A recreated durable starts from its configured `Offset`, like a re-declared queue starts empty. +One teardown case deliberately does NOT end `Subscribe`: a client-initiated +`Disconnect` while its consume stream is still open. Ending the subscription +would end the caller's whole consume stream, but the amqp091 connector +leaves that stream open — its subscribe loop just goes quiet once the AMQP +connection closes — so a client that disconnects and then acks straggler +in-flight messages gets each ack answered with a "could not retrieve broker +details" failure rather than end-of-stream. `Subscribe` blocks until the +stream's own context ends; delivery cannot resume either way, because the +disconnect already stopped the consume machinery and drained the connection. + ## Configuration | Environment variable | Default | Meaning | diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index f16b329..db474ed 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -146,6 +146,14 @@ type natsBrokerDetails struct { state atomic.Uint32 consumed int64 produced int64 + + // clientDisconnect distinguishes a client-initiated Disconnect from the + // NATS library closing the connection terminally (both read state CLOSED): + // a disconnected client's Subscribe parks until its stream ends instead of + // ending the stream. Set before the consume contexts are stopped, so any + // Subscribe woken by that teardown observes it. Mirrors amqp091's flag of + // the same name. + clientDisconnect atomic.Bool } // durableFilterUse records the shared header-filter fingerprint of a durable's @@ -376,7 +384,9 @@ func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetai if bd := p.getBrokerDetailsByIdentifier(clientID); bd != nil { return bd, nil } - return nil, fmt.Errorf("could not retrieve broker details for connection: %s", clientID) + // Error string matches the amqp091 connector: clients (and the + // integration suite) observe it through the ack path after a disconnect. + return nil, fmt.Errorf("could not retrieve broker details for this connection: %s", clientID) } func (p *natsjsProvider) getBrokerDetailsByIdentifier(id string) *natsBrokerDetails { @@ -505,7 +515,9 @@ func (p *natsjsProvider) Disconnect(ctx context.Context) { } // Mark the connection closed before stopping its consume contexts: // stopping one wakes its Subscribe (Closed() fires), which reads the - // state to tell deliberate teardown from a consumer lost on the server. + // state to tell deliberate teardown from a consumer lost on the server — + // and the flag to tell a client's own Disconnect from a terminal close. + bd.clientDisconnect.Store(true) bd.state.Store(provider.CLOSED) for _, name := range bd.consumeContexts.GetList() { if cc, ok := bd.consumeContexts.Get(name); ok { @@ -1054,6 +1066,22 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // fires for this connection's own teardown (Disconnect stops every // consume context, Drain closes the subscriptions), which is not an // error — the context or connection state says which case this is. + if bd.clientDisconnect.Load() && ctx.Err() == nil { + // Client-initiated Disconnect with the consume stream still + // open: do not end the subscription — returning here would end + // the caller's whole consume stream, but the amqp091 connector + // leaves it open (its subscribe loop just goes quiet once the + // AMQP connection closes), so a client that disconnects and + // then acks its in-flight messages gets each ack answered with + // a "could not retrieve broker details" failure rather than + // end-of-stream. Delivery cannot resume — the consume context + // is stopped and the connection is draining — so just wait for + // the client to close its stream. Scoped to the client's own + // Disconnect (not any CLOSED state): a connection the library + // closed terminally still ends the subscription. + <-ctx.Done() + return nil + } if ctx.Err() != nil || bd.state.Load() == provider.CLOSED { return nil } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 24d8b91..c05e880 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -112,6 +112,49 @@ func TestConnectDisconnect(t *testing.T) { assert.False(t, p.ClientExists("test-client")) } +// TestSubscribeOutlivesDisconnect: a client-initiated Disconnect must not end +// a live Subscribe — ending it ends the caller's whole consume stream, but +// the amqp091 connector leaves that stream open (its subscribe loop goes +// quiet when the AMQP connection closes), so a client that disconnects and +// then acks straggler messages gets per-ack failures, not end-of-stream. +// Subscribe may only return once the subscription's own context ends. +func TestSubscribeOutlivesDisconnect(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + src := queueSource("events.disc.q", "events.disc", "e") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan *pb.Error, 1) + go func() { done <- p.Subscribe(subCtx, src, make(chan *pb.Message, 1)) }() + + require.Eventually(t, func() bool { + stream, serr := bd.js.Stream(ctx, streamNameFor("events.disc")) + if serr != nil { + return false + } + info, ierr := stream.Info(ctx) + return ierr == nil && info.State.Consumers == 1 + }, 10*time.Second, 20*time.Millisecond, "subscription never established") + + p.Disconnect(ctx) + + select { + case serr := <-done: + t.Fatalf("Subscribe ended on Disconnect (%v); it must stay open until the stream context ends", serr) + case <-time.After(2 * time.Second): + } + + cancel() + select { + case serr := <-done: + assert.Nil(t, serr, "a disconnected subscription ends cleanly with its context") + case <-time.After(10 * time.Second): + t.Fatal("Subscribe did not return after its context ended") + } +} + // TestConcurrentConnectClosesRedundantLink: the gRPC server's "already // connected?" check is not atomic, so two Connect calls for one client // identifier can both reach the provider. Only one connection may survive; the From 15ae30a7a9219d007a26ba420cfef65e1e49f09c Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Sun, 19 Jul 2026 21:43:48 -0400 Subject: [PATCH 53/77] fix: require a declared stream for unary publishes to a STREAM address PublishOne to a STREAM address auto-created the backing stream, where amqp091 routes unary stream publishes through the RabbitMQ Streams client, which refuses a stream nobody declared (declaring is the reader's job - streamSubscribe's DeclareStream is what creates it). The divergence is a silent-success footgun: a publisher with a typo'd or never-declared stream address got success, and its messages accumulated in a junk stream no reader will ever see, instead of the immediate error RabbitMQ gives. The unary path now looks the stream up instead of ensuring it: a miss answers "stream ... does not exist" and creates nothing, and the no-stream-response self-heal re-checks existence rather than resurrecting a stream deleted out from under the connection (a RabbitMQ stream publisher errors after deletion too). A hit shares the per-connection memo with the ensure path. The streaming Publish path deliberately keeps auto-creating: amqp091 sends every address type there, STREAM included, over an auto-declared exchange and reports no error, so refusing it would diverge in the other direction - and storing the message beats dropping it into an unbound exchange. Message-contract refusals (dedup on a queue address, PublishId without PublisherName) now precede the stream assertion, as they do on amqp091, so a publish that is wrong twice over reports the contract error rather than the missing stream, and a refused publish creates no topology. The design doc's topology-mapping section now states when streams come into being and this exception. Integration TestProduceFailsIfNoStream and TestProduceStreamWithDeduplicationFailWithOutProducerName both pass against JetStream. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 ++ internal/provider/connectors/natsjs/natsjs.go | 84 +++++++++++++---- .../provider/connectors/natsjs/natsjs_test.go | 89 +++++++++++++++++++ 3 files changed, 166 insertions(+), 16 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 3188ab1..4ac5dce 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -42,6 +42,15 @@ an empty routing key maps to the bare prefix `events.orders.~`. A JetStream stream named `arke_
` captures `
.~` and `
.~.>`, and each consumer filters on the mapped source subjects. +The stream for an address is asserted on use: a subscribe (or declare-only +call) creates it, and so does a publish — with one exception. A unary publish +to a `STREAM` address requires the stream to exist already, exactly as +amqp091's stream publisher refuses a stream nobody declared: declaring a +stream is its readers' job, and auto-creating it would turn a typo'd address +name into a junk stream storing messages no reader will ever see. The +streaming publish path does not check, matching amqp091, which sends every +address type over an auto-declared exchange without error. + Stream names (and durable consumer names, which JetStream validates the same way) may not contain `.`, whitespace, `*`, `>`, `/` or `\`, so they are derived from the address (or source / consumer-group) name by swapping dots diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index db474ed..25cddfd 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -54,6 +54,13 @@ const ( queueDedupError = "Message deduplication is not supported by Queue types" ) +// streamMissingError answers a unary publish to a STREAM address whose stream +// was never declared. amqp091's wording there is stream-client plumbing +// ("failed to create a stream publisher"), so instead of copying it this takes +// the RabbitMQ stream client's canonical phrase; the contract a client relies +// on is that the publish errors rather than creating the stream. +const streamMissingError = "publish: stream %q does not exist" + const ( // defaultAckWait is how long the server waits for an ack before // redelivering a message. See ackWait for the trade-off it sets; @@ -622,6 +629,26 @@ func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDeta return streamName, nil } +// lookupStream answers the stream name backing an address only if the stream +// already exists on the server; it never creates one. The unary publish path +// uses it for STREAM addresses, where declaring the stream is the reader's job +// (see PublishOne). A hit is memoized alongside ensureStreamFor's entries: +// either way the connection has established that the stream exists, which is +// all the memo records. +func (p *natsjsProvider) lookupStream(ctx context.Context, bd *natsBrokerDetails, addr *pb.Address) (string, error) { + streamName := streamNameFor(addr.GetName()) + if _, ok := bd.knownStreams.Get(streamName); ok { + return streamName, nil + } + lctx, cancel := context.WithTimeout(ctx, jsAPITimeout()) + defer cancel() + if _, err := bd.js.Stream(lctx, streamName); err != nil { + return "", err + } + bd.knownStreams.Add(streamName, true) + return streamName, nil +} + // boundSources merges bindings into the stream's existing source set, so a // second address-to-address binding adds to the first rather than replacing // it. Bindings accumulate the same way on RabbitMQ, where declaring one never @@ -699,7 +726,7 @@ func (p *natsjsProvider) Publish(ctx context.Context, in <-chan *pb.Message, err errChan <- &pb.Error{Message: unsupportedConfirmError} continue } - errChan <- p.publishMsg(ctx, bd, msg) + errChan <- p.publishMsg(ctx, bd, msg, false) } } } @@ -709,22 +736,25 @@ func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Er if err != nil { return &pb.Error{Message: err.Error(), IsFatal: true} } - return p.publishMsg(ctx, bd, msg) + // A STREAM address must name a stream that already exists: amqp091 routes + // unary stream publishes through the RabbitMQ Streams client, which + // refuses a stream nobody declared (declaring is the reader's job — + // streamSubscribe's DeclareStream). Auto-creating here instead would turn + // a typo'd address name into a junk stream that swallows messages no + // reader will ever see. Only this unary path checks: amqp091's streaming + // Publish sends every address type, STREAM included, over an + // auto-declared exchange and reports no error either, so the streaming + // path keeps the auto-create. + requireDeclared := msg.GetAddress().GetType() == pb.Address_STREAM + return p.publishMsg(ctx, bd, msg, requireDeclared) } -func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message) *pb.Error { +func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message, requireDeclared bool) *pb.Error { addr := msg.GetAddress() - streamName, err := p.ensureStreamFor(ctx, bd, addr) - if err != nil { - return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} - } - nmsg := &nats.Msg{ - Subject: publishSubjectFor(addr.GetName(), firstSubject(addr)), - Data: msg.GetBody(), - Header: pbToNatsHeader(msg.GetHeaders()), - } - var pubOpts []jetstream.PublishOpt - // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). + // Message-contract refusals come before any topology is touched: amqp091 + // refuses both of these before it reaches for the stream or exchange, so + // a publish that is wrong twice over reports the contract error, not the + // missing stream — and a refused publish creates nothing. if msg.GetPublishId() > 0 && addr.GetType() == pb.Address_QUEUE { // JetStream would happily deduplicate this, since dedup here is a // property of the stream rather than of the address type. Refusing it @@ -737,6 +767,24 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, if msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} } + assertStream := p.ensureStreamFor + if requireDeclared { + assertStream = p.lookupStream + } + streamName, err := assertStream(ctx, bd, addr) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return &pb.Error{Message: fmt.Sprintf(streamMissingError, addr.GetName())} + } + if err != nil { + return &pb.Error{Message: fmt.Sprintf("ensure stream: %s", err.Error())} + } + nmsg := &nats.Msg{ + Subject: publishSubjectFor(addr.GetName(), firstSubject(addr)), + Data: msg.GetBody(), + Header: pbToNatsHeader(msg.GetHeaders()), + } + var pubOpts []jetstream.PublishOpt + // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). if msg.GetPublishId() > 0 { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) } @@ -749,10 +797,14 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, // publish — and a NATS client outlives broker state changes that would // sever an AMQP connection. Drop the entry, re-assert the stream, and // retry once; the first attempt was not stored (no stream answered), - // so the retry cannot duplicate it. + // so the retry cannot duplicate it. For a STREAM address the + // re-assertion is the existence check again, so a stream deleted out + // from under the connection is reported missing, not resurrected. bd.knownStreams.Delete(streamName) - if _, rerr := p.ensureStreamFor(ctx, bd, addr); rerr == nil { + if _, rerr := assertStream(ctx, bd, addr); rerr == nil { _, perr = bd.js.PublishMsg(ctx, nmsg, pubOpts...) + } else if errors.Is(rerr, jetstream.ErrStreamNotFound) { + return &pb.Error{Message: fmt.Sprintf(streamMissingError, addr.GetName())} } } if perr != nil { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index c05e880..984de89 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -516,6 +516,85 @@ func TestPublishRecreatesDeletedStream(t *testing.T) { assert.Equal(t, uint64(1), stream.CachedInfo().State.Msgs, "the retried publish must be stored") } +// TestPublishOneStreamAddressRequiresDeclaredStream pins the amqp091 contract +// for unary publishes to a STREAM address: the stream must already have been +// declared by a reader (amqp091 routes these through the RabbitMQ Streams +// client, which refuses a stream nobody declared), so a typo'd address name +// errors instead of silently minting a junk stream that swallows the +// messages. +func TestPublishOneStreamAddressRequiresDeclaredStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.undeclared", Type: pb.Address_STREAM, Subjects: []string{"e"}} + perr := p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")}) + require.NotNil(t, perr, "publishing to an undeclared stream must fail") + assert.Equal(t, fmt.Sprintf(streamMissingError, "events.undeclared"), perr.GetMessage()) + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("events.undeclared")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, "the refused publish must not create the stream") + + // Once a reader has declared the stream, the same publish succeeds. + _, eerr := p.ensureStream(ctx, bd, "events.undeclared") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) +} + +// TestPublishOneStreamAddressDeletedStreamNotRecreated is the STREAM-address +// counterpart of TestPublishRecreatesDeletedStream: the self-heal that +// re-ensures an auto-created stream must not resurrect one the operator +// deleted, because for a STREAM address the stream is the reader's declared +// topology — a RabbitMQ stream publisher errors after deletion too. +func TestPublishOneStreamAddressDeletedStreamNotRecreated(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dropped", Type: pb.Address_STREAM, Subjects: []string{"e"}} + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dropped") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("before")})) + + require.NoError(t, bd.js.DeleteStream(ctx, streamNameFor("events.dropped"))) + + perr := p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("after")}) + require.NotNil(t, perr, "a deleted stream must fail the publish, not be resurrected") + assert.Equal(t, fmt.Sprintf(streamMissingError, "events.dropped"), perr.GetMessage()) + _, serr := bd.js.Stream(ctx, streamNameFor("events.dropped")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound) +} + +// TestPublishStreamingPathStillAutoCreatesStreamAddress pins the deliberate +// split: only the unary path requires a declared stream. amqp091's streaming +// Publish sends STREAM addresses over an auto-declared exchange and reports +// no error either, so the streaming path keeps auto-creating — the message is +// stored where amqp091 would drop it into an unbound exchange. +func TestPublishStreamingPathStillAutoCreatesStreamAddress(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 1) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + in <- &pb.Message{ + Address: &pb.Address{Name: "events.streamed", Type: pb.Address_STREAM, Subjects: []string{"e"}}, + Body: []byte("m")} + select { + case e := <-errChan: + require.Nil(t, e, "the streaming publish must auto-create the stream: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.streamed")) + require.NoError(t, serr, "the stream must have been auto-created") + assert.Equal(t, uint64(1), stream.CachedInfo().State.Msgs) +} + func TestSubscribeRecreatesDeletedStream(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -1858,6 +1937,16 @@ func TestPublishContractRefusals(t *testing.T) { Address: &pb.Address{Name: "events.t", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, PublisherName: "pub", PublishId: 1, Body: []byte("x")})) + // A message wrong twice over — dedup contract violated AND aimed at an + // undeclared stream — reports the contract error: amqp091 refuses both + // of these before it reaches for the stream, so the refusals precede the + // stream lookup here too. + perr = p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.tcr.undeclared", Type: pb.Address_STREAM, Subjects: []string{"k"}}, + PublishId: 7, Body: []byte("x")}) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "PublisherName not set on message") + // Confirm on the fire-and-forget streaming publish. in := make(chan *pb.Message, 1) errChan := make(chan *pb.Error, 2) From 7ad868c45aaf883ce3328b2f110adb785102a92b Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Mon, 20 Jul 2026 11:57:05 -0400 Subject: [PATCH 54/77] docs: record why dead-lettering republishes instead of using advisories The advisory-driven DLQ (a stream capturing $JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES / MSG_TERMINATED) is the common NATS idiom for this, so document why the connector does not use it: an advisory names the origin stream_seq rather than carrying the message, so a dead-letter consumer would need a second fetch to see the body, and the record dangles once the primary stream's MaxAge evicts the original. Republishing keeps the DLQ an ordinary address an unmodified AMQP client can consume. Also document that redelivery is unbounded. RabbitMQ quorum queues carry a delivery limit (20 by default on 4.x); this connector sets no MaxDeliver deliberately, because RabbitMQ's delayed-retry idiom republishes through a TTL queue and a DLX and so resets the delivery count, while Retry here is a NakWithDelay that increments the same NumDelivered a limit would cap. Matching the number would cut off retries RabbitMQ allows without bound. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 51 ++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 4ac5dce..b6c6493 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -328,6 +328,34 @@ same message deduplicates in the DLQ within its dedup window, and the broker-side move preserves the death trail in `x-death`, and the retry count is this connector's equivalent (a plain republish would lose it). +### Rejected alternative: advisory-driven dead-lettering + +JetStream publishes advisories when a consumer exhausts `MaxDeliver` +(`$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES..`) or when a +message is terminated (`…MSG_TERMINATED.…`), and a common NATS idiom builds a +"dead-letter queue" as an ordinary stream capturing those subjects. That +idiom was considered and rejected for this connector. + +An advisory carries metadata, not the message: its payload names the origin +`stream_seq`, so a reader has to fetch the original out of the primary stream +to see the body or headers. A dead-letter consumer would then have to speak +two protocols — advisory JSON, then a stream fetch — where its AMQP +counterpart just consumes the dead-lettered message off a queue. It also +inherits the primary stream's retention: once `MaxAge` evicts the original, +the advisory still resolves to nothing, so the dead-letter record decays into +a dangling pointer exactly when it is most likely to be read. + +Republishing the message itself keeps the DLQ a normal address that an +unmodified AMQP client can consume, preserves the body, headers and +`x-retry-count`, and gives the dead-letter copy retention independent of the +original. The cost is that the move is two proxy-side steps rather than one +broker-side one, which the ordering above is designed to make safe. + +The advisory subjects would become relevant only if this connector set +`MaxDeliver`, as the hook for routing a delivery-exhausted message into the +same republish path instead of letting the consumer silently skip it. It does +not set `MaxDeliver` — see the redelivery limitation below. + ## Retry-count header Clients that count retries via an `x-retry-count` header (rather than @@ -648,9 +676,28 @@ amqp091 connector, so existing client sources validate unchanged: durable's is therefore rejected (subject filters, which the server enforces, still update the shared consumer in place). - **Dead-letter is a proxy-side republish.** The DLQ is an ordinary stream - fed by the connector; there is no advisory-driven re-consumption. A failed - DLQ publish fails the dead-letter call — the message stays in flight and is + fed by the connector; there is no advisory-driven re-consumption (see + "Rejected alternative: advisory-driven dead-lettering"). A failed DLQ + publish fails the dead-letter call — the message stays in flight and is redelivered — rather than dropping the message. +- **Redelivery is unbounded.** The connector does not set `MaxDeliver`, so a + message that is delivered and then neither acked nor nacked — a consumer + that wedges, or crashes mid-handler in a loop — is redelivered every + `NATSJS_ACK_WAIT` for as long as the subscription lives. RabbitMQ quorum + queues apply a delivery limit (20 by default on 4.x) and, on reaching it, + dead-letter the message if the queue has a dead-letter exchange or drop it + if it does not. The difference is deliberate: the two brokers count + deliveries differently, because RabbitMQ's delayed-retry idiom republishes + the message through a TTL queue and a dead-letter exchange, which resets its + delivery count, while `Retry` here is a `NakWithDelay` that increments the + same `NumDelivered` a delivery limit would cap. A `MaxDeliver` chosen to + match RabbitMQ's limit would therefore cut off *retries* that RabbitMQ + allows without bound — breaking clients whose retry policy deliberately + retries far more than 20 times — and separating the two cases would need + per-message state the broker does not keep. Retaining the message is also + the safer divergence: nothing is lost, and the stall is visible as the + consumer's `num_pending`. Deployments that want a ceiling should alert on + redelivery rate rather than ask the broker to discard. - **Transient sources are per-subscriber.** A transient (auto-delete / exclusive / TEMPORARY) source maps to an ephemeral consumer created per subscription, so two clients subscribing with the same transient source From 3d2f5a1cb68db83bfaf3a20bfd14c8516f0f7727 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 13:10:13 -0400 Subject: [PATCH 55/77] fix: keep address-to-address bindings when a declare carries none CreateOrUpdateStream replaces a stream's source set wholesale, but ensureStreamFor only populated it when the current call had bindings of its own to declare. Any assertion that reached the create path without them therefore unbound every address-to-address binding another subscriber had declared on that stream. The call that does this is the most ordinary one there is: a publisher naming a bound address directly carries no ParentAddress, so it arrives with no bindings. The per-connection knownStreams memo hides it from the connection that declared the binding, which is why the existing coverage missed it -- the regression only appears from a second connection, where routing from the parent silently stops with no error anywhere. Read the live source set on every assertion and write it back, so an assertion that declares nothing preserves what is already there. Both paths now do a read-modify-write, so both are serialized under bindMu; the parent's own stream is asserted before that lock is taken, since the nested assertion runs through the same lock. boundSources loses its I/O and becomes the pure merge it always was, and the stream info it reads is the one Stream() already fetched. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 6 +- internal/provider/connectors/natsjs/natsjs.go | 85 ++++++++++++------ .../provider/connectors/natsjs/natsjs_test.go | 88 +++++++++++++++++++ 3 files changed, 150 insertions(+), 29 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index b6c6493..86c8770 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -158,7 +158,11 @@ accumulate: declaring one never removes another, as on RabbitMQ, so a second subscriber binding a different key on the same address adds to the set rather than replacing it. Nothing unbinds — a binding outlives the subscription that declared it, as an AMQP exchange-to-exchange binding outlives the client that -bound it. +bound it. That holds for assertions carrying no binding at all, which is the +ordinary case: a publisher names the bound address directly and so has no +parent to declare. Because JetStream replaces a stream's source set wholesale +on update, every assertion reads the current set and writes it back, rather +than sending only what the caller happens to know about. This subject scheme — and the stream/durable name encoding above — is part of the connector's persistence contract: retained messages are stored under diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 25cddfd..5bf185a 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -574,6 +574,17 @@ func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDeta return streamName, nil } } + // The parent's own stream is asserted before bindMu is taken below: that + // assertion runs through the same lock, so holding it across the nested + // call would deadlock. + var parentStream string + if len(bindings) > 0 { + ps, perr := p.ensureStream(ctx, bd, addr.GetParentAddress().GetName()) + if perr != nil { + return "", fmt.Errorf("ensure parent address stream: %w", perr) + } + parentStream = ps + } build := func() error { cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jsAPITimeout()) defer cancel() @@ -594,33 +605,41 @@ func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDeta MaxAge: streamMaxAge(), MaxBytes: streamMaxBytes(), } + // The source set is carried forward on EVERY assertion, not just the + // ones that declare a binding of their own: CreateOrUpdateStream + // replaces it wholesale, so a config that omits it unbinds every + // address-to-address binding another subscriber declared on this + // stream. Nothing here unbinds (see mergeBoundSources), and an + // ordinary publisher to a bound address — which names no parent, so it + // reaches this path with no bindings — must not be the thing that does. + sources, serr := p.existingSources(cctx, bd, streamName) + if serr != nil { + return serr + } if len(bindings) > 0 { - parentStream, perr := p.ensureStream(cctx, bd, addr.GetParentAddress().GetName()) - if perr != nil { - return fmt.Errorf("ensure parent address stream: %w", perr) - } - sources, serr := p.boundSources(cctx, bd, streamName, parentStream, bindings) - if serr != nil { - return serr - } - cfg.Sources = sources + sources = mergeBoundSources(sources, parentStream, bindings) } + cfg.Sources = sources _, err := bd.js.CreateOrUpdateStream(cctx, cfg) return err } + // build reads the stream's current binding set and writes back a superset, + // so the read-modify-write has to be serialized or one concurrent caller's + // bindings are lost — including a caller that only meant to assert the + // stream exists. The streams registry cannot do this job: it *collapses* + // concurrent callers, which is right when they all want the same stream but + // would silently drop one of two different binding sets. var err error if len(bindings) > 0 { - // boundSources reads the current binding set to add to it, so the - // read-modify-write has to be serialized or a concurrent subscriber - // declaring a different binding would be overwritten. The streams - // registry cannot do this job: it *collapses* concurrent callers, which - // is right when they all want the same stream but would silently drop - // one of two different binding sets. p.bindMu.Lock() err = build() p.bindMu.Unlock() } else { - err = p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), build) + err = p.streams.ensure(ctx, streamEnsureKey(bd.connectionConfig, streamName), func() error { + p.bindMu.Lock() + defer p.bindMu.Unlock() + return build() + }) } if err != nil { return "", err @@ -649,22 +668,32 @@ func (p *natsjsProvider) lookupStream(ctx context.Context, bd *natsBrokerDetails return streamName, nil } -// boundSources merges bindings into the stream's existing source set, so a +// existingSources reads the address-to-address bindings already configured on a +// stream, or nil if the stream does not exist yet. Stream() itself fetches the +// stream info, so CachedInfo costs nothing further. +func (p *natsjsProvider) existingSources(ctx context.Context, bd *natsBrokerDetails, + streamName string) ([]*jetstream.StreamSource, error) { + st, err := bd.js.Stream(ctx, streamName) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read existing bindings: %w", err) + } + if info := st.CachedInfo(); info != nil { + return info.Config.Sources, nil + } + return nil, nil +} + +// mergeBoundSources merges bindings into a stream's existing source set, so a // second address-to-address binding adds to the first rather than replacing // it. Bindings accumulate the same way on RabbitMQ, where declaring one never // removes another; nothing here unbinds, so a binding outlives the // subscription that declared it, exactly as an AMQP exchange-to-exchange // binding outlives the client that bound it. -func (p *natsjsProvider) boundSources(ctx context.Context, bd *natsBrokerDetails, - streamName, parentStream string, bindings []string) ([]*jetstream.StreamSource, error) { - sources := []*jetstream.StreamSource{} - if st, err := bd.js.Stream(ctx, streamName); err == nil { - if info, ierr := st.Info(ctx); ierr == nil { - sources = info.Config.Sources - } - } else if !errors.Is(err, jetstream.ErrStreamNotFound) { - return nil, fmt.Errorf("read existing bindings: %w", err) - } +func mergeBoundSources(sources []*jetstream.StreamSource, + parentStream string, bindings []string) []*jetstream.StreamSource { var from *jetstream.StreamSource for _, s := range sources { if s.Name == parentStream { @@ -688,7 +717,7 @@ func (p *natsjsProvider) boundSources(ctx context.Context, bd *natsBrokerDetails from.SubjectTransforms = append(from.SubjectTransforms, jetstream.SubjectTransformConfig{Source: subject, Destination: subject}) } - return sources, nil + return sources } func firstSubject(addr *pb.Address) string { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 984de89..1efde1d 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -2797,3 +2797,91 @@ func TestExpiresThreshold(t *testing.T) { _, err = expiresThreshold(src(map[string]string{"Expires": "-5"})) require.EqualError(t, err, "value for Expires option must be a positive integer") } + +// connectSecondClient attaches a second, independently-identified client to the +// same server, so a test can exercise what one connection's state (the +// knownStreams memo, say) hides from another. +func connectSecondClient(t *testing.T, s *natsserver.Server, id string) (*natsjsProvider, context.Context) { + t.Helper() + saved := GetClientIdentifier + t.Cleanup(func() { GetClientIdentifier = saved }) + GetClientIdentifier = func(context.Context) (string, error) { return id, nil } + + addr := s.Addr().(*net.TCPAddr) + p := NewNATSJetStreamProvider().(*natsjsProvider) + ctx := context.Background() + if perr := p.Connect(ctx, &pb.ConnectionConfiguration{ + Host: "127.0.0.1", Port: int32(addr.Port), //nolint:gosec // local server port fits int32 + ClientName: id}, false); perr != nil { + t.Fatalf("connect %s: %s", id, perr.GetMessage()) + } + if !p.WaitForConnect(ctx) { + t.Fatalf("WaitForConnect returned false for %s", id) + } + return p, ctx +} + +// TestBindingSurvivesBindinglessDeclare: asserting a stream that has +// address-to-address bindings must carry them forward even when the asserting +// call declares none of its own. CreateOrUpdateStream replaces the source set +// wholesale, so a config that omits it silently unbinds the address — and the +// call that omits it is the most ordinary one there is: a publisher naming the +// bound address directly, which carries no ParentAddress. The per-connection +// knownStreams memo hides this from the connection that declared the binding, +// so the regression only shows up from a second one. +func TestBindingSurvivesBindinglessDeclare(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.bsurv.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.bsurv.child", + Type: pb.Address_TOPIC, + Subjects: []string{"key.one"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{Name: "events.bsurv.consumer", Type: pb.Source_QUEUE, + Address: child, Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + + pubToParent := func(t *testing.T, body string) { + t.Helper() + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.bsurv.parent", Subjects: []string{"key.one"}}, + Body: []byte(body)})) + } + + pubToParent(t, "before") + m := recv(t, out) + require.Equal(t, "before", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + // A second client publishes straight to the bound address, naming no + // parent. Its connection has never asserted this stream, so it takes the + // create-or-update path with no bindings of its own to declare. + p2, ctx2 := connectSecondClient(t, s, "other-client") + require.Nil(t, p2.PublishOne(ctx2, &pb.Message{ + Address: &pb.Address{Name: "events.bsurv.child", Subjects: []string{"direct"}}, + Body: []byte("direct")})) + + // The binding the first client declared must still route. + GetClientIdentifier = func(context.Context) (string, error) { return "test-client", nil } + pubToParent(t, "after") + + deadline := time.After(10 * time.Second) + for { + select { + case m := <-out: + require.Nil(t, p.Ack(ctx, m.GetUuid())) + if string(m.GetBody()) == "after" { + return + } + case <-deadline: + t.Fatal("the address-to-address binding did not survive a bindingless declare") + } + } +} From 9dbfad4d1eaf1c8a08a0dc1547c92f7281d99465 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 13:10:41 -0400 Subject: [PATCH 56/77] fix: pace the redelivery that retries a failed dead-letter A dead-letter that fails leaves the message in flight so the server's fallback nack retries it rather than losing it. That nack was immediate, and the retry runs the same dead-letter attempt against the same configuration -- so a failure that never clears spins the message between server and client as fast as the client can nack it, with nothing to stop it: MaxDeliver is deliberately left unset so a slow consumer is never silently cut off. The cheapest way in is an empty DeadLetterAddress. The gRPC server enters DeadLetter whenever the option key is present, whatever its value, so an empty one reaches the connector, fails every time, and loops. Measured on an embedded server: over 300 deliveries a second of a single message, indefinitely. A DeadLetterAddress that cannot be resolved into a stream behaves the same way. Delay the redelivery instead (NATSJS_DEADLETTER_RETRY_DELAY, default 5s). A transient failure still recovers promptly; a permanent one costs one redelivery per interval instead of hundreds a second. Keeping the message rather than dropping it is unchanged -- terminating it would lose it while it is also absent from any DLQ. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 15 ++++- internal/provider/connectors/natsjs/natsjs.go | 30 +++++++++- .../provider/connectors/natsjs/natsjs_test.go | 57 +++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 86c8770..3fc25e1 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -325,7 +325,19 @@ the message ack-pending — the server then falls back to a nack, so the message is redelivered and dead-lettering is retried instead of the message being lost. A present but empty `DeadLetterAddress` is treated the same way: the connector returns an error and leaves the original in flight rather than -terminating it without a DLQ publish. The DLQ copy carries a `Nats-Msg-Id` +terminating it without a DLQ publish. + +That fallback nack is delayed by `NATSJS_DEADLETTER_RETRY_DELAY` (default 5s) +rather than issued immediately. The retry runs the same dead-letter attempt +against the same configuration, so a failure that does not clear on its own — +a `DeadLetterAddress` that cannot be resolved into a stream, or one set to the +empty string — would otherwise spin the message between server and client as +fast as the client can nack it, and nothing bounds that loop (`MaxDeliver` is +deliberately left unset so a slow consumer is never silently cut off). Pacing +it keeps a transient failure recovering promptly while a permanent one costs +one redelivery per interval instead of hundreds a second. + +The DLQ copy carries a `Nats-Msg-Id` derived from the original's stream sequence, so a retried dead-letter of the same message deduplicates in the DLQ within its dedup window, and the `x-retry-count` the consumer saw when it gave the message up — RabbitMQ's @@ -559,6 +571,7 @@ disconnect already stopped the consume machinery and drained the connection. | `NATSJS_API_TIMEOUT` | `30s` | Deadline for JetStream management API calls (stream / consumer creation). Go duration. | | `NATSJS_ACK_WAIT` | `30s` | How long the server waits for an ack before redelivering a message. Go duration. | | `NATSJS_SAC_PINNED_TTL` | `1m` | Single-active-consumer failover deadline: how long the pinned client may go without pulling before a standby takes over. Go duration. | +| `NATSJS_DEADLETTER_RETRY_DELAY` | `5s` | How long to wait before redelivering a message whose dead-letter attempt failed, so a permanently-failing dead-letter cannot spin. Go duration. | TLS and credentials come from the standard `ConnectionConfiguration` (`Tls`, `Credentials`) and the server's `tlsSkipVerify` flag, exactly as for the AMQP diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 5bf185a..933dad2 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -98,6 +98,10 @@ const ( // from the pinned (active) client before unpinning and failing over to a // standby. See sacPinnedTTL; override via NATSJS_SAC_PINNED_TTL. defaultSACPinnedTTL = time.Minute + // defaultDeadLetterRetryDelay paces the redelivery that retries a failed + // dead-letter. See deadLetterRetryDelay; override via + // NATSJS_DEADLETTER_RETRY_DELAY. + defaultDeadLetterRetryDelay = 5 * time.Second ) // supportedSourceOptionsList intentionally matches the amqp091 connector so that @@ -383,6 +387,27 @@ func sacPinnedTTL() time.Duration { return defaultSACPinnedTTL } +// deadLetterRetryDelay paces the redelivery that retries a dead-letter attempt +// which failed with the message still in flight (see DeadLetter and Nack). +// That redelivery has to be delayed rather than immediate: the retry runs the +// same dead-letter attempt against the same configuration, so a failure that +// does not clear on its own — a DeadLetterAddress that cannot be resolved into +// a stream, or one set to the empty string — otherwise spins the message +// between server and client as fast as the client can nack it, and nothing +// bounds the loop (MaxDeliver is deliberately left unset so a slow consumer is +// never silently cut off). Delaying it keeps a transient failure recovering +// promptly while a permanent one costs one redelivery per interval instead of +// hundreds a second. Configured via NATSJS_DEADLETTER_RETRY_DELAY (Go +// duration). +func deadLetterRetryDelay() time.Duration { + if v := os.Getenv("NATSJS_DEADLETTER_RETRY_DELAY"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultDeadLetterRetryDelay +} + func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { clientID, err := GetClientIdentifier(ctx) if err != nil { @@ -1523,7 +1548,10 @@ func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { return perr } if redeliver { - if err := m.Nak(); err != nil { + // Delayed, not immediate: this redelivery is a dead-letter retry, and + // retrying a permanently-failing dead-letter as fast as the client can + // nack is a redelivery storm nothing bounds. See deadLetterRetryDelay. + if err := m.NakWithDelay(deadLetterRetryDelay()); err != nil { return &pb.Error{Message: err.Error()} } return nil diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 1efde1d..a763630 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -385,6 +385,10 @@ func TestDeadLetterPreservesRoutingKey(t *testing.T) { // caller's fallback nack still resolves the uuid and the message is // redelivered — the failure mode is retry, not silent loss. func TestDeadLetterFailureKeepsMessage(t *testing.T) { + // The fallback nack of a failed dead-letter is paced (see + // deadLetterRetryDelay); shorten it so the test does not wait it out. + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "100ms") + s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -420,6 +424,10 @@ func TestDeadLetterFailureKeepsMessage(t *testing.T) { } func TestDeadLetterEmptyAddressKeepsMessage(t *testing.T) { + // The fallback nack of a failed dead-letter is paced (see + // deadLetterRetryDelay); shorten it so the test does not wait it out. + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "100ms") + s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -2885,3 +2893,52 @@ func TestBindingSurvivesBindinglessDeclare(t *testing.T) { } } } + +// TestFailedDeadLetterRedeliveryIsPaced: a dead-letter that fails leaves the +// message in flight so the server's fallback nack retries it (see +// TestDeadLetterFailureKeepsMessage). That retry runs the same attempt against +// the same configuration, so when the failure is permanent — here an empty +// DeadLetterAddress — an immediate nak spins the message between server and +// client as fast as the client can nack, and MaxDeliver is deliberately unset +// so nothing bounds it. The redelivery must be paced instead. +func TestFailedDeadLetterRedeliveryIsPaced(t *testing.T) { + t.Setenv("NATSJS_DEADLETTER_RETRY_DELAY", "500ms") + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.dlpace", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 64) + src := queueSource("events.dlpace.consumer", "events.dlpace", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + // Replay the gRPC server's nack handling for one poison message: it calls + // DeadLetter whenever the DeadLetterAddress key is present (whatever its + // value) and falls back to Nack when that errors. + deliveries := 0 + const window = 3 * time.Second + deadline := time.After(window) + for { + select { + case m := <-out: + deliveries++ + if deliveries > 40 { + t.Fatalf("redelivery storm: %d deliveries of one message in under %s", deliveries, window) + } + require.NotNil(t, p.DeadLetter(ctx, src, m.GetUuid())) + require.Nil(t, p.Nack(ctx, m.GetUuid())) + case <-deadline: + // Paced, but still retrying: dropping the message instead would + // lose it while it is also absent from any DLQ. + assert.GreaterOrEqual(t, deliveries, 2, "a failed dead-letter must still be retried") + assert.LessOrEqual(t, deliveries, 20, + "redelivery of a permanently-failing dead-letter must be paced, not immediate") + return + } + } +} From 729eff71fc8bf000a2da39ed512f631fbad3f05f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 13:11:05 -0400 Subject: [PATCH 57/77] fix: stop a redelivery mark from resurrecting a resolved message Resolving a message (take, then delete) and marking one for redelivery (read, then update) are both read-modify-writes over activeMessages. The map is individually concurrency-safe, but nothing serialized the pairs, so a mark landing between a concurrent resolve's read and its delete reinserted an entry nothing could ever resolve -- leaked for the life of the connection and counted in the active-message stat. Exercising the two concurrently over 300 messages left 138 orphans. Serialize the pairs under a dedicated mutex; the network-bound naks in releaseInFlight are moved outside it. Two smaller items found in the same sweep: An unrecognized Offset was rejected only after ensureStreamFor had already created the address's stream, leaving topology behind for an address the client only ever named by mistake. Validate it with the other contract refusals, before any topology is touched -- the ordering Expires and the single-active ConsumerGroup check already follow. The durable header-filter conflict check is per connection, so it cannot see two clients on separate connections attaching to one durable. That is the topology it is nominally protecting, so record what it does and does not guarantee rather than leaving it to read as complete. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 52 ++++++++++++++++--- .../provider/connectors/natsjs/natsjs_test.go | 48 +++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 933dad2..dcd8d01 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -127,6 +127,13 @@ type natsBrokerDetails struct { // Ack/Nack/Retry/DeadLetter RPC can resolve it and teardown can release // only the closing subscription's own deliveries. activeMessages *util.ConcurrentMap + // activeMu serializes the read-modify-write pairs over activeMessages — + // the take-then-delete that resolves a message, and the read-then-update + // that marks one for redelivery. The map is individually concurrency-safe, + // but without this a mark landing between a concurrent take's read and its + // delete would reinsert an entry nothing can ever resolve, leaking it for + // the life of the connection and inflating the active-message stat. + activeMu sync.Mutex // knownStreams memoizes streams we have already ensured. knownStreams *util.ConcurrentMap // consumeContexts holds each live subscription's jetstream.ConsumeContext @@ -151,6 +158,16 @@ type natsBrokerDetails struct { // live subscribers share and how many hold it, so a conflicting declaration // is rejected instead of silently dropping messages. Subject filters are // excluded — they are the server's and update in place. + // + // Scope: this map is per connection, so the check catches conflicting + // subscribers that share one, not two clients on separate connections + // attaching to the same durable — the server-side consumer is shared but + // nothing here is. Enforcing it across connections would mean putting the + // fingerprint somewhere both can read (consumer metadata), which buys + // little: subscribers of one durable are ordinarily replicas of one + // service declaring identical filters, so a disagreement is a client bug + // this catches only when it is cheap to. Treat it as a guard rail, not a + // guarantee. durableFilters map[string]*durableFilterUse durableFilterMu sync.Mutex @@ -907,6 +924,13 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c IsFatal: true, } } + // Also validated before any topology is created: an unrecognized Offset is + // a refusal, and a refused subscribe must not leave a stream behind for an + // address the client only ever named by mistake. + deliverPolicy, startSeq, derr := deliverPolicyFor(source) + if derr != nil { + return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), derr.Error()), IsFatal: true} + } addr := source.GetAddress() streamName, serr := p.ensureStreamFor(ctx, bd, addr) if serr != nil { @@ -946,10 +970,6 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c if prefetch <= 0 { prefetch = -1 } - deliverPolicy, startSeq, derr := deliverPolicyFor(source) - if derr != nil { - return &pb.Error{Message: fmt.Sprintf("source %q: %s", source.GetName(), derr.Error()), IsFatal: true} - } consCfg := jetstream.ConsumerConfig{ FilterSubjects: filterSubjectsFor(source), AckPolicy: jetstream.AckExplicitPolicy, @@ -1233,6 +1253,10 @@ func consumerLostError(source *pb.Source, consumerName string, err error) *pb.Er // resolves concurrently with teardown is deleted by whichever side gets there // first; the loser's ack/nak fails and is ignored (at-least-once either way). func releaseInFlight(bd *natsBrokerDetails, subKey string, nak bool) { + // Claim the entries under activeMu (so a concurrent mark cannot reinsert + // one), then do the network-bound naks outside it. + released := make(map[string]jetstream.Msg) + bd.activeMu.Lock() for _, uuid := range bd.activeMessages.GetList() { mu, ok := bd.activeMessages.Get(uuid) if !ok { @@ -1243,10 +1267,14 @@ func releaseInFlight(bd *natsBrokerDetails, subKey string, nak bool) { continue } bd.activeMessages.Delete(uuid) - if !nak { - continue - } - if err := im.msg.Nak(); err != nil { + released[uuid] = im.msg + } + bd.activeMu.Unlock() + if !nak { + return + } + for uuid, msg := range released { + if err := msg.Nak(); err != nil { util.Logger.Debugf("natsjs: nak of in-flight message %s at subscription end: %s", uuid, err.Error()) } } @@ -1480,6 +1508,8 @@ func (p *natsjsProvider) takeMessage(ctx context.Context, uuid string) (jetstrea if err != nil { return nil, &pb.Error{Message: err.Error()} } + bd.activeMu.Lock() + defer bd.activeMu.Unlock() mu, ok := bd.activeMessages.Get(uuid) if !ok { return nil, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} @@ -1495,6 +1525,8 @@ func (p *natsjsProvider) takeMessageForNack(ctx context.Context, uuid string) (j if err != nil { return nil, false, &pb.Error{Message: err.Error()} } + bd.activeMu.Lock() + defer bd.activeMu.Unlock() mu, ok := bd.activeMessages.Get(uuid) if !ok { return nil, false, &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} @@ -1509,6 +1541,8 @@ func (p *natsjsProvider) takeMessageForNack(ctx context.Context, uuid string) (j // with the message still in flight: the error it returns makes the server nack // the same uuid, and that nack is the retry, not a rejection. func markForRedelivery(bd *natsBrokerDetails, uuid string) { + bd.activeMu.Lock() + defer bd.activeMu.Unlock() mu, ok := bd.activeMessages.Get(uuid) if !ok { return @@ -1678,7 +1712,9 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid if err := m.Term(); err != nil { return &pb.Error{Message: err.Error()} } + bd.activeMu.Lock() bd.activeMessages.Delete(uuid) + bd.activeMu.Unlock() return nil } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index a763630..99bfdff 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -2942,3 +2942,51 @@ func TestFailedDeadLetterRedeliveryIsPaced(t *testing.T) { } } } + +// TestSubscribeInvalidOffsetCreatesNoStream: an unrecognized Offset is a +// contract refusal, and a refused subscribe must not leave topology behind for +// an address the client only ever named by mistake. +func TestSubscribeInvalidOffsetCreatesNoStream(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + perr := p.Subscribe(ctx, streamSource("events.nostream.consumer", "events.nostream", "latest", "e"), out) + require.NotNil(t, perr) + assert.Contains(t, perr.GetMessage(), "invalid offset") + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("events.nostream")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, + "a subscribe refused on its options must not have created the address's stream") +} + +// TestConcurrentResolveAndMarkLeavesNoOrphans: resolving a message and marking +// one for redelivery are both read-modify-writes over activeMessages. Run +// concurrently without a lock between them, a mark landing between a resolve's +// read and its delete reinserts an entry nothing can ever resolve, leaking it +// for the life of the connection and inflating the active-message stat. +func TestConcurrentResolveAndMarkLeavesNoOrphans(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + const n = 300 + uuids := make([]string, 0, n) + for i := 0; i < n; i++ { + uuid := fmt.Sprintf("uuid-%d", i) + uuids = append(uuids, uuid) + bd.activeMessages.Add(uuid, inflightMsg{subKey: "sub#1"}) + } + + var wg sync.WaitGroup + for _, uuid := range uuids { + wg.Add(2) + go func() { defer wg.Done(); _, _ = p.takeMessage(ctx, uuid) }() + go func() { defer wg.Done(); markForRedelivery(bd, uuid) }() + } + wg.Wait() + + assert.Equal(t, 0, bd.activeMessages.Length(), + "marking a message for redelivery must not reinsert one that was resolved concurrently") +} From 8ed0939067009157b98b52e958c8ed9da9582180 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 14:23:47 -0400 Subject: [PATCH 58/77] fix: refuse a publish id on every non-stream address, not just a queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deduplication is a STREAM-address feature on both brokers. amqp091's PublishOne routes only Address_STREAM to publishOneStream — the RabbitMQ Streams client, the one thing that can deduplicate — and sends every other type to publishOneQueue, which refuses a publish id outright. The connector only refused Address_QUEUE, so a TOPIC or FILTER address silently accepted the id and deduplicated on it. TOPIC is the case that matters: it is the protobuf zero value, so an address that never sets a type took the accepting path. A client developing against natsjs would get deduplication there and then a hard error the moment the same code ran against RabbitMQ — the exact cross-broker divergence the refusal exists to prevent. Scope the refusal to the unary path while widening it by address type, because amqp091 splits the same way: its streaming Publish hands every non-STREAM address to a plain channel publish that neither refuses the id nor deduplicates on it. So the streaming path now ignores the id rather than honouring it — refusing would be stricter than RabbitMQ and break a publisher that works there today, while honouring it hands the client a guarantee that disappears on cutover. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 16 +++ internal/provider/connectors/natsjs/natsjs.go | 69 +++++++----- .../provider/connectors/natsjs/natsjs_test.go | 102 +++++++++++++++++- 3 files changed, 157 insertions(+), 30 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 3fc25e1..aff8edc 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -389,6 +389,22 @@ connector); the connector sets `Nats-Msg-Id` from that pair so re-publishes within the window are collapsed. A publisher name by itself does not enable deduplication. +Dedup belongs to `STREAM` addresses, and only to them. JetStream would happily +deduplicate on any address — dedup here is a property of the stream, not of +the address type — but RabbitMQ gets it from the Streams client, which is the +only thing the AMQP connector reaches for a `STREAM` address. So the two paths +mirror that connector rather than what JetStream can do: + +- a **unary publish** (`PublishOne`) carrying a publish id for any other + address type — `TOPIC`, `QUEUE` or `FILTER` — is refused, because the AMQP + connector refuses it. `TOPIC` is the case to watch: it is the protobuf zero + value, so an address that never sets a type is refused too; +- a **streaming publish** carrying one *ignores* it, because the AMQP + connector's streaming path hands every non-`STREAM` address to a plain + channel publish that drops the id on the floor. Refusing would be stricter + than RabbitMQ, and honouring it would hand the client a guarantee that + disappears the moment it runs there. + ## Retention: work-queue vs append-log This is the most important behavioral difference between the two brokers. A diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index dcd8d01..375a4d1 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -797,7 +797,7 @@ func (p *natsjsProvider) Publish(ctx context.Context, in <-chan *pb.Message, err errChan <- &pb.Error{Message: unsupportedConfirmError} continue } - errChan <- p.publishMsg(ctx, bd, msg, false) + errChan <- p.publishMsg(ctx, bd, msg, false /* unary */) } } } @@ -807,39 +807,50 @@ func (p *natsjsProvider) PublishOne(ctx context.Context, msg *pb.Message) *pb.Er if err != nil { return &pb.Error{Message: err.Error(), IsFatal: true} } - // A STREAM address must name a stream that already exists: amqp091 routes - // unary stream publishes through the RabbitMQ Streams client, which - // refuses a stream nobody declared (declaring is the reader's job — - // streamSubscribe's DeclareStream). Auto-creating here instead would turn - // a typo'd address name into a junk stream that swallows messages no - // reader will ever see. Only this unary path checks: amqp091's streaming - // Publish sends every address type, STREAM included, over an - // auto-declared exchange and reports no error either, so the streaming - // path keeps the auto-create. - requireDeclared := msg.GetAddress().GetType() == pb.Address_STREAM - return p.publishMsg(ctx, bd, msg, requireDeclared) + return p.publishMsg(ctx, bd, msg, true /* unary */) } -func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message, requireDeclared bool) *pb.Error { +// publishMsg publishes one message. unary distinguishes PublishOne from the +// streaming Publish, which amqp091 routes through entirely different client +// code — so two parts of the contract below differ by path, not just by +// address type. +func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, msg *pb.Message, unary bool) *pb.Error { addr := msg.GetAddress() + // Dedup is a STREAM-address feature on both brokers: RabbitMQ gets it from + // the Streams client, which is the only thing amqp091's publish paths + // reach for a STREAM address. + dedupAddress := addr.GetType() == pb.Address_STREAM // Message-contract refusals come before any topology is touched: amqp091 - // refuses both of these before it reaches for the stream or exchange, so - // a publish that is wrong twice over reports the contract error, not the - // missing stream — and a refused publish creates nothing. - if msg.GetPublishId() > 0 && addr.GetType() == pb.Address_QUEUE { - // JetStream would happily deduplicate this, since dedup here is a - // property of the stream rather than of the address type. Refusing it - // anyway keeps the option meaning one thing across brokers: amqp091 - // cannot offer dedup on a queue address (RabbitMQ dedup comes from - // streams), so a client allowed to set it here would be writing code - // that silently loses its deduplication on RabbitMQ. + // refuses these before it reaches for the stream or exchange, so a publish + // that is wrong twice over reports the contract error, not the missing + // stream — and a refused publish creates nothing. + if msg.GetPublishId() > 0 && unary && !dedupAddress { + // The refusal covers every non-STREAM address, not just Address_QUEUE: + // amqp091's PublishOne routes STREAM to publishOneStream and everything + // else — TOPIC, QUEUE and FILTER alike — to publishOneQueue, which is + // where the refusal lives (amqp091.go:1501). TOPIC matters most: it is + // the proto zero value, so an address that never sets a type takes this + // path. JetStream would happily deduplicate any of them, since dedup + // here is a property of the stream rather than of the address type; + // refusing anyway keeps the option meaning one thing across brokers, + // instead of letting a client write a publish that works here and + // fails outright on RabbitMQ. return &pb.Error{Message: queueDedupError} } if msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} } + // A STREAM address must name a stream that already exists: amqp091 routes + // unary stream publishes through the RabbitMQ Streams client, which + // refuses a stream nobody declared (declaring is the reader's job — + // streamSubscribe's DeclareStream). Auto-creating here instead would turn + // a typo'd address name into a junk stream that swallows messages no + // reader will ever see. Only the unary path checks: amqp091's streaming + // Publish sends every address type, STREAM included, over an + // auto-declared exchange and reports no error either, so the streaming + // path keeps the auto-create. assertStream := p.ensureStreamFor - if requireDeclared { + if unary && dedupAddress { assertStream = p.lookupStream } streamName, err := assertStream(ctx, bd, addr) @@ -855,8 +866,14 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, Header: pbToNatsHeader(msg.GetHeaders()), } var pubOpts []jetstream.PublishOpt - // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window). - if msg.GetPublishId() > 0 { + // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window), + // but only where amqp091 would also have deduplicated. On a non-STREAM + // address the id is ignored rather than honoured: amqp091's streaming + // Publish hands every non-STREAM address to the plain channel publish, + // which drops the id on the floor, so applying it here would silently give + // the client a guarantee that disappears on RabbitMQ. (The unary path never + // gets this far with that combination — it is refused above.) + if msg.GetPublishId() > 0 && dedupAddress { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) } _, perr := bd.js.PublishMsg(ctx, nmsg, pubOpts...) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 99bfdff..00a8ce5 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -875,7 +875,14 @@ func TestDeduplication(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) - addr := &pb.Address{Name: "events.dedup", Subjects: []string{"e"}} + // Dedup is a STREAM-address feature on both brokers — RabbitMQ gets it from + // the Streams client, which is the only thing amqp091's publish paths reach + // for a STREAM address — and a unary publish to one needs a declared stream. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup") + require.NoError(t, eerr) + + addr := &pb.Address{Name: "events.dedup", Type: pb.Address_STREAM, Subjects: []string{"e"}} dup := func() *pb.Message { return &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} } @@ -894,8 +901,15 @@ func TestPublishIDRequiresPublisherName(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) + // A STREAM address: the only kind whose unary publish reaches the + // PublisherName check at all, since every other type is refused outright + // for asking for dedup (see TestPublishDedupRefusedOnEveryNonStreamAddress). + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup.required") + require.NoError(t, eerr) + perr := p.PublishOne(ctx, &pb.Message{ - Address: &pb.Address{Name: "events.dedup.required", Subjects: []string{"e"}}, + Address: &pb.Address{Name: "events.dedup.required", Type: pb.Address_STREAM, Subjects: []string{"e"}}, Body: []byte("x"), PublishId: 1, }) @@ -1940,8 +1954,10 @@ func TestPublishContractRefusals(t *testing.T) { require.NotNil(t, perr) assert.Equal(t, queueDedupError, perr.GetMessage()) - // ...but the same publish to a non-queue address is fine. - require.Nil(t, p.PublishOne(ctx, &pb.Message{ + // ...and equally on a topic address. amqp091's PublishOne sends every + // non-STREAM address through publishOneQueue, which is where that refusal + // lives, so TOPIC and FILTER are refused on exactly the same grounds. + require.NotNil(t, p.PublishOne(ctx, &pb.Message{ Address: &pb.Address{Name: "events.t", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, PublisherName: "pub", PublishId: 1, Body: []byte("x")})) @@ -2990,3 +3006,81 @@ func TestConcurrentResolveAndMarkLeavesNoOrphans(t *testing.T) { assert.Equal(t, 0, bd.activeMessages.Length(), "marking a message for redelivery must not reinsert one that was resolved concurrently") } + +// TestPublishDedupRefusedOnEveryNonStreamAddress pins the address types a +// unary publish may ask for deduplication on. amqp091's PublishOne routes only +// Address_STREAM to publishOneStream (the RabbitMQ Streams client, the one +// thing that can deduplicate); every other type — TOPIC, QUEUE and FILTER — +// goes to publishOneQueue, which refuses PublishId outright (amqp091.go:1501). +// TOPIC is the case that matters most: it is the proto zero value, so an +// address that never sets a type lands here. +// +// Regression: the refusal used to test == Address_QUEUE, so a TOPIC or FILTER +// address silently accepted the id and deduplicated — giving a client a +// guarantee that turns into a hard error the moment it runs against RabbitMQ. +func TestPublishDedupRefusedOnEveryNonStreamAddress(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + for _, tc := range []struct { + name string + typ pb.Address_TargetType + }{ + {"topic (the proto zero value)", pb.Address_TOPIC}, + {"queue", pb.Address_QUEUE}, + {"filter", pb.Address_FILTER}, + } { + t.Run(tc.name, func(t *testing.T) { + perr := p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.scope." + tc.name, Type: tc.typ, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")}) + require.NotNil(t, perr, "a unary publish asking for dedup on a non-STREAM address must be refused") + assert.Equal(t, queueDedupError, perr.GetMessage()) + }) + } + + // A STREAM address is the one that may: declare it, then dedup applies. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, eerr := p.ensureStream(ctx, bd, "events.dedup.scope.stream") + require.NoError(t, eerr) + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dedup.scope.stream", Type: pb.Address_STREAM, Subjects: []string{"k"}}, + PublisherName: "pub", PublishId: 1, Body: []byte("x")})) +} + +// TestStreamingPublishIgnoresPublishIDOnNonStreamAddress pins the other half of +// the dedup contract: the refusal belongs to the unary path only. amqp091's +// streaming Publish hands every non-STREAM address to the plain channel +// publish, which neither refuses the id nor deduplicates on it — so natsjs +// must not refuse it either (that would be stricter than RabbitMQ, breaking a +// publisher that works there today), and must not honour it either (that would +// be a guarantee RabbitMQ does not give). +func TestStreamingPublishIgnoresPublishIDOnNonStreamAddress(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 2) + errChan := make(chan *pb.Error, 2) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + + addr := &pb.Address{Name: "events.stream.dedupignored", Type: pb.Address_TOPIC, Subjects: []string{"k"}} + for i := 0; i < 2; i++ { + in <- &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} + select { + case e := <-errChan: + require.Nil(t, e, "the streaming path must not refuse a publish id: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } + } + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.stream.dedupignored.consumer", + Address: &pb.Address{Name: "events.stream.dedupignored"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(2), stats.GetMessageCount(), + "the id must be ignored on a non-STREAM address, not deduplicated on — RabbitMQ drops it on the floor") +} From f899fc8b15125607929ad6f49a2abb918415c264 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 14:24:57 -0400 Subject: [PATCH 59/77] fix: stop a headers address filtering on the routing key it must ignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A headers exchange routes on header arguments alone. RabbitMQ matches every one of an exchange's bindings on those arguments and never reads the routing key, so the binding keys amqp091 passes to QueueBind and ExchangeBind for an Address_FILTER decide nothing at all. The connector turned those keys into NATS consumer filter subjects, so a message published to a headers address under a key no binding named was dropped — delivering strictly less than RabbitMQ would, silently. The no-subjects case already had this right (a headers address with filters selects the whole address); it was only the with-subjects case that narrowed. Select the whole address for a headers address either way and let evaluateFilters decide, which is what it is there for. The keys still travel to the parent in an address-to-address binding, where the parent's type decides how they are matched — so a topic parent goes on matching them as routing keys, and a headers parent ignores them in turn. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 8 ++ .../provider/connectors/natsjs/helpers.go | 22 +++++- .../connectors/natsjs/helpers_test.go | 41 ++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 75 +++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index aff8edc..b53f1ae 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -138,6 +138,14 @@ the escaping above can never emit. Three cases are deliberately not that: single `""` key there purely to have somewhere to hang the header arguments; `evaluateFilters` is this connector's stand-in for those. +A headers address selects the whole address whether or not it declares +binding keys, for that same reason: a headers exchange matches every one of +its bindings on their header arguments alone and never reads the routing key, +so keys declared on one decide nothing and must not narrow the consumer. +They still travel to the parent in an address-to-address binding, where the +*parent's* type decides how they are matched — a topic parent matches them as +routing keys, a headers parent ignores them in turn. + ### Address-to-address binding An address may name a `ParentAddress`, binding it to that parent: what is diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 9439086..227ed29 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -387,6 +387,17 @@ func ownFilterSubjectsFor(source *pb.Source) []string { if len(source.GetAddress().GetSubjects()) == 0 { return boundNothingSubjects(source) } + // A headers address routes on headers alone: RabbitMQ's headers exchange + // never looks at the routing key, so the subjects a source declares on one + // are binding keys the broker discards (amqp091 passes them to QueueBind / + // ExchangeBind all the same, and they decide nothing). Selecting the whole + // address and letting evaluateFilters decide is the equivalent — filtering + // by subject here would drop messages RabbitMQ delivers, which is the same + // reasoning boundNothingSubjects already applies to a headers address that + // carries no subjects at all. + if source.GetAddress().GetType() == pb.Address_FILTER { + return wholeAddressSubjects(source.GetAddress().GetName()) + } if source.GetAddress().GetType() == pb.Address_QUEUE { return directFilterSubjectsFor(source) } @@ -467,13 +478,22 @@ func pruneSubsumed(subjects []string) []string { // single character other than "~e". const unmatchableToken = "~x" +// wholeAddressSubjects selects everything an address's stream captures — the +// bare prefix (empty routing key) and every subject under it. It is the filter +// set for a source bound to the address as a whole rather than to particular +// routing keys. +func wholeAddressSubjects(addressName string) []string { + prefix := subjectPrefix(addressName) + return []string{prefix, prefix + ".>"} +} + // boundNothingSubjects gives the filter subjects for a source whose address // carries no routing-key patterns. amqp091 declares no binding at all in that // case (declareBinding iterates an empty subject list), so the source receives // nothing — with two exceptions that both mean "the whole address". func boundNothingSubjects(source *pb.Source) []string { prefix := subjectPrefix(source.GetAddress().GetName()) - whole := []string{prefix, prefix + ".>"} + whole := wholeAddressSubjects(source.GetAddress().GetName()) // A stream source is not bound to its address at all. amqp091 reads a // RabbitMQ stream by name — streamSubscribe goes straight to the stream // connection and never declares an exchange or a binding — so its reader diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index d34e47d..456e31a 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -443,3 +443,44 @@ func TestEvaluateFilters(t *testing.T) { // multiple filters are OR'd: second filter matches assert.True(t, evaluateFilters([]*pb.Filter{allMiss, all}, headers)) } + +// TestFilterSubjectsForHeadersAddress pins that a headers address selects its +// whole subject space regardless of the subjects it declares: RabbitMQ's +// headers exchange routes on header arguments alone and never reads the +// routing key, so binding subjects decide nothing there. evaluateFilters is +// the connector's stand-in for the header arguments. +func TestFilterSubjectsForHeadersAddress(t *testing.T) { + whole := []string{"events.hdr.~", "events.hdr.~.>"} + + withSubjects := &pb.Source{Type: pb.Source_QUEUE, Address: &pb.Address{ + Name: "events.hdr", Type: pb.Address_FILTER, Subjects: []string{"bound.one", "bound.two"}}} + assert.Equal(t, whole, filterSubjectsFor(withSubjects), + "subjects on a headers address must not narrow the filter") + + // Already the behaviour when it carries no subjects but has filters, and + // the two must agree. + noSubjects := &pb.Source{Type: pb.Source_QUEUE, + Address: &pb.Address{Name: "events.hdr", Type: pb.Address_FILTER}, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, + Matches: []*pb.Match{{Name: "tenant", Value: "acme"}}}}} + assert.Equal(t, whole, filterSubjectsFor(noSubjects)) + + // A topic address with the same subjects still narrows — the widening is + // scoped to headers addresses. + topic := &pb.Source{Type: pb.Source_QUEUE, Address: &pb.Address{ + Name: "events.hdr", Type: pb.Address_TOPIC, Subjects: []string{"bound.one"}}} + assert.Equal(t, []string{"events.hdr.~.bound.one"}, filterSubjectsFor(topic)) +} + +// TestParentBindingSubjectsHeadersParent: the binding keys of an +// address-to-address binding are matched by the PARENT exchange's type, so a +// headers parent ignores them too — amqp091's ExchangeBind passes the child's +// subjects with no arguments, which a headers exchange matches unconditionally. +func TestParentBindingSubjectsHeadersParent(t *testing.T) { + addr := &pb.Address{ + Name: "events.child", + Subjects: []string{"bound.one"}, + ParentAddress: &pb.Address{Name: "events.parent", Type: pb.Address_FILTER}, + } + assert.Equal(t, []string{"events.parent.~", "events.parent.~.>"}, parentBindingSubjects(addr)) +} diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 00a8ce5..0eeb50b 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3084,3 +3084,78 @@ func TestStreamingPublishIgnoresPublishIDOnNonStreamAddress(t *testing.T) { assert.Equal(t, int64(2), stats.GetMessageCount(), "the id must be ignored on a non-STREAM address, not deduplicated on — RabbitMQ drops it on the floor") } + +// TestHeadersAddressIgnoresRoutingKey covers a headers address that also +// carries subjects. RabbitMQ's headers exchange never looks at the routing key +// — rabbit_exchange_type_headers matches every binding of the exchange on its +// header arguments alone — so the subjects amqp091 passes to QueueBind decide +// nothing, and a message published under any key reaches the consumer. +// +// Regression: subjects on a FILTER address used to become NATS consumer filter +// subjects, so a message whose routing key matched no subject was dropped — +// silently delivering strictly less than RabbitMQ would. +func TestHeadersAddressIgnoresRoutingKey(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.headers", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{Name: "events.headers.consumer", Type: pb.Source_QUEUE, + Address: addr, Options: map[string]string{"Offset": "first"}}, out) + time.Sleep(500 * time.Millisecond) + + for _, key := range []string{"bound.one", "some.other.key"} { + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers", Type: pb.Address_FILTER, Subjects: []string{key}}, + Body: []byte(key)})) + } + + got := map[string]bool{} + for len(got) < 2 { + m := recv(t, out) + got[string(m.GetBody())] = true + require.Nil(t, p.Ack(ctx, m.GetUuid())) + } + assert.True(t, got["bound.one"], "the message under the declared key must arrive") + assert.True(t, got["some.other.key"], + "a headers exchange ignores routing keys, so a message under any key must arrive") +} + +// TestHeadersAddressStillAppliesHeaderFilters is the guard on the test above: +// selecting the whole address must hand the decision to the header filters, +// not deliver everything unconditionally. +func TestHeadersAddressStillAppliesHeaderFilters(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.headers.filtered.consumer", Type: pb.Source_QUEUE, Address: addr, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, Matches: []*pb.Match{{Name: "tenant", Value: "acme"}}}}, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + // Matching headers under a key no subject names: delivered (headers decide). + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"any.key"}}, + Headers: map[string]string{"tenant": "acme"}, Body: []byte("match")})) + // Non-matching headers under the declared key: filtered out. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.headers.filtered", Type: pb.Address_FILTER, Subjects: []string{"bound.one"}}, + Headers: map[string]string{"tenant": "other"}, Body: []byte("nomatch")})) + + m := recv(t, out) + assert.Equal(t, "match", string(m.GetBody())) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("header filters must still gate a headers address; got %q", string(extra.GetBody())) + case <-time.After(2 * time.Second): + } +} From 45d1eaf13d7483f9a295f01c9c033ca249b9e61f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 14:26:10 -0400 Subject: [PATCH 60/77] fix: assert a stream only when the assertion would change it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because JetStream replaces a stream's source set wholesale on update, every assertion reads the current set and writes it back — a read-modify-write serialized by p.bindMu. That lock is process-local, and Arke runs more than one replica: two processes interleaving a read and a write on one stream can still drop a binding the other just added, which is the failure bindMu was introduced to prevent within a process. Compare the live configuration against the one built and return early when the stream already satisfies it. That takes the overwhelmingly common case out of the race entirely — a bare assertion against a steady-state stream is every publisher's first touch and every reconnect, and it now reads and stops without writing. What remains is callers that genuinely change the configuration, which for bindings are symmetric: both are adding. A stream created before a limit existed is still retrofitted, since its configuration differs. Two details the comparison has to get right. The config being built needs its own copy of the source set: mergeBoundSources appends to a StreamSource's transforms in place, so sharing them with the snapshot would mutate the very thing being compared, the merge would equal itself, and a newly declared binding would be skipped instead of written. Only a second binding onto one parent exposes that — a first appends a whole element and changes the length. And JetStream normalizes an unset size limit to its -1 sentinel and echoes that back, so a config asking for 0 describes the same stream; comparing the two literally made every assertion look like a change and defeated the skip entirely. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 13 ++ internal/provider/connectors/natsjs/natsjs.go | 151 ++++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 206 ++++++++++++++++++ 3 files changed, 360 insertions(+), 10 deletions(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index b53f1ae..7c03945 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -172,6 +172,19 @@ parent to declare. Because JetStream replaces a stream's source set wholesale on update, every assertion reads the current set and writes it back, rather than sending only what the caller happens to know about. +Reading and writing back makes an assertion a read-modify-write, and the lock +that serializes it is process-local while Arke runs more than one replica. So +an assertion writes only when it would actually change something: it compares +the live configuration against the one it built and returns early if the +stream already satisfies it. That takes the overwhelmingly common case — a +bare assertion against a steady-state stream, which is every publisher's +first touch and every reconnect — out of the race entirely, leaving only +callers that genuinely change the configuration, which for bindings are +symmetric (both are adding). A stream created before a limit existed is still +retrofitted: its configuration differs, so the update runs. Two concurrent +*binding* declarations across replicas remain a real, much narrower window; +closing it would need a compare-and-swap JetStream does not offer. + This subject scheme — and the stream/durable name encoding above — is part of the connector's persistence contract: retained messages are stored under these subjects for the life of the stream's retention limits, so any future diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 375a4d1..a6e1bd0 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -654,14 +654,42 @@ func (p *natsjsProvider) ensureStreamFor(ctx context.Context, bd *natsBrokerDeta // stream. Nothing here unbinds (see mergeBoundSources), and an // ordinary publisher to a bound address — which names no parent, so it // reaches this path with no bindings — must not be the thing that does. - sources, serr := p.existingSources(cctx, bd, streamName) + existing, serr := p.existingConfig(cctx, bd, streamName) if serr != nil { return serr } + if existing != nil { + // Its OWN copy, independent of the snapshot in existing: + // mergeBoundSources appends to a StreamSource's transforms in + // place, so sharing them would mutate the very thing the config is + // about to be compared against — the merge would compare equal to + // itself and a newly declared binding would be skipped instead of + // written. Only visible when the parent is already in the source + // set (the second binding onto one parent), since a first binding + // appends a fresh element and changes the length. + cfg.Sources = copySources(existing.Sources) + } if len(bindings) > 0 { - sources = mergeBoundSources(sources, parentStream, bindings) + cfg.Sources = mergeBoundSources(cfg.Sources, parentStream, bindings) + } + // Assert only when the assertion would actually change something. + // Carrying the source set forward makes this a read-modify-write, and + // bindMu serializes that only within one process — Arke runs more than + // one replica, so two processes interleaving a read and a write on one + // stream can still drop a binding the other just added. Skipping the + // write when the live config already satisfies this caller removes the + // overwhelmingly common case from that race entirely: a bare assertion + // against a steady-state stream (every publisher's first touch, every + // reconnect) now reads and stops. What remains is two callers that both + // genuinely change the config, which is far rarer and, for bindings, + // symmetric — both are adding. + // + // A stream created before these limits existed still gets retrofitted: + // its MaxAge/MaxBytes differ, so it does not satisfy the config and the + // update runs. + if existing != nil && streamConfigSatisfied(existing, &cfg) { + return nil } - cfg.Sources = sources _, err := bd.js.CreateOrUpdateStream(cctx, cfg) return err } @@ -710,24 +738,127 @@ func (p *natsjsProvider) lookupStream(ctx context.Context, bd *natsBrokerDetails return streamName, nil } -// existingSources reads the address-to-address bindings already configured on a -// stream, or nil if the stream does not exist yet. Stream() itself fetches the -// stream info, so CachedInfo costs nothing further. -func (p *natsjsProvider) existingSources(ctx context.Context, bd *natsBrokerDetails, - streamName string) ([]*jetstream.StreamSource, error) { +// existingConfig reads a stream's live configuration — its address-to-address +// bindings and the limits an assertion may need to update — or nil if the +// stream does not exist yet. Stream() itself fetches the stream info, so +// CachedInfo costs nothing further. +func (p *natsjsProvider) existingConfig(ctx context.Context, bd *natsBrokerDetails, + streamName string) (*jetstream.StreamConfig, error) { st, err := bd.js.Stream(ctx, streamName) if errors.Is(err, jetstream.ErrStreamNotFound) { return nil, nil } if err != nil { - return nil, fmt.Errorf("read existing bindings: %w", err) + return nil, fmt.Errorf("read existing stream config: %w", err) } if info := st.CachedInfo(); info != nil { - return info.Config.Sources, nil + cfg := info.Config + // Deep-copy the source set. mergeBoundSources appends to a + // StreamSource's transforms in place, and the value CachedInfo hands + // back shares those pointers — so without this the merged config and + // the config it is compared against would be the same objects, + // streamConfigSatisfied would always agree, and a newly declared + // binding would be silently skipped instead of written. + cfg.Sources = copySources(cfg.Sources) + return &cfg, nil } return nil, nil } +func copySources(in []*jetstream.StreamSource) []*jetstream.StreamSource { + if in == nil { + return nil + } + out := make([]*jetstream.StreamSource, 0, len(in)) + for _, s := range in { + if s == nil { + continue + } + c := *s + c.SubjectTransforms = slices.Clone(s.SubjectTransforms) + out = append(out, &c) + } + return out +} + +// streamConfigSatisfied reports whether a stream's live config already provides +// everything an assertion asks for, so the assertion can skip its update (see +// ensureStreamFor). Only the fields the connector sets are compared; anything +// an operator tuned outside them is left alone, exactly as CreateOrUpdateStream +// would leave it. +func streamConfigSatisfied(live, want *jetstream.StreamConfig) bool { + if live.Storage != want.Storage || live.Retention != want.Retention || + live.Duplicates != want.Duplicates || live.Replicas != want.Replicas || + live.MaxAge != want.MaxAge || !sameLimit(live.MaxBytes, want.MaxBytes) { + return false + } + if !sameStringSet(live.Subjects, want.Subjects) { + return false + } + return sameSources(live.Sources, want.Sources) +} + +// sameLimit compares two JetStream size/count limits, treating every +// non-positive value as the same "unlimited". The server normalizes an unset +// limit to -1 and echoes it back that way, so a config asking for 0 describes +// the same stream as a live -1 — comparing them literally would make every +// assertion look like a change. +func sameLimit(live, want int64) bool { + if live <= 0 && want <= 0 { + return true + } + return live == want +} + +func sameStringSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + seen := make(map[string]int, len(a)) + for _, s := range a { + seen[s]++ + } + for _, s := range b { + seen[s]-- + if seen[s] < 0 { + return false + } + } + return true +} + +// sameSources compares two address-to-address binding sets by the fields +// mergeBoundSources builds: the parent stream each sources from and the +// subjects it routes in. Order is irrelevant on both levels — the server may +// return either in any order. +func sameSources(live, want []*jetstream.StreamSource) bool { + if len(live) != len(want) { + return false + } + byName := make(map[string]*jetstream.StreamSource, len(live)) + for _, s := range live { + byName[s.Name] = s + } + for _, w := range want { + l, ok := byName[w.Name] + if !ok { + return false + } + if !sameStringSet(transformSubjects(l), transformSubjects(w)) { + return false + } + } + return true +} + +func transformSubjects(s *jetstream.StreamSource) []string { + out := make([]string, 0, len(s.SubjectTransforms)) + for _, t := range s.SubjectTransforms { + out = append(out, t.Source+">"+t.Destination) + } + return out +} + // mergeBoundSources merges bindings into a stream's existing source set, so a // second address-to-address binding adds to the first rather than replacing // it. Bindings accumulate the same way on RabbitMQ, where declaring one never diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 0eeb50b..6a46ca1 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -14,6 +14,7 @@ import ( "time" natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" pb "github.com/sassoftware/arke/api" "github.com/stretchr/testify/assert" @@ -3159,3 +3160,208 @@ func TestHeadersAddressStillAppliesHeaderFilters(t *testing.T) { case <-time.After(2 * time.Second): } } + +// TestBareStreamAssertionIssuesNoUpdate pins that asserting a stream that +// already satisfies the requested config writes nothing. +// +// Carrying the existing binding set forward makes ensureStreamFor a +// read-modify-write, and p.bindMu serializes that only inside one process — +// Arke runs more than one replica. Skipping the write when nothing would +// change takes the overwhelmingly common case (every publisher's first touch +// of an existing stream) out of that cross-process race, so a bare assertion +// can no longer be the thing that drops a binding another replica just added. +func TestBareStreamAssertionIssuesNoUpdate(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + bd := p.getBrokerDetailsByIdentifier("test-client") + + streamName := streamNameFor("events.noupdate") + _, eerr := p.ensureStream(ctx, bd, "events.noupdate") + require.NoError(t, eerr) + + // Watch the JetStream management API for updates to this stream. A plain + // subscription is an observer: the server's own responder still receives + // the request, so this changes nothing about the call itself. + updates := make(chan struct{}, 8) + sub, serr := bd.nc.Subscribe("$JS.API.STREAM.UPDATE."+streamName, func(*nats.Msg) { + updates <- struct{}{} + }) + require.NoError(t, serr) + defer sub.Unsubscribe() //nolint:errcheck + require.NoError(t, bd.nc.Flush()) + + // Drop the per-connection memo so the assertion really re-runs against the + // server, exactly as a fresh connection's first touch would. + bd.knownStreams.Delete(streamName) + _, eerr = p.ensureStream(ctx, bd, "events.noupdate") + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + + select { + case <-updates: + t.Fatal("a bare assertion of an already-satisfying stream must not issue a stream update") + case <-time.After(500 * time.Millisecond): + } + + // ...but an assertion that genuinely changes something still writes. A + // binding is the case that matters: it must survive the skip logic. + child := &pb.Address{ + Name: "events.noupdate", + Type: pb.Address_TOPIC, + Subjects: []string{"routed.key"}, + ParentAddress: &pb.Address{Name: "events.noupdate.parent", Type: pb.Address_TOPIC}, + } + _, eerr = p.ensureStreamFor(ctx, bd, child) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + case <-time.After(2 * time.Second): + t.Fatal("declaring a new binding must still update the stream") + } + + // And the binding really landed — the skip must not have swallowed it. + st, ferr := bd.js.Stream(ctx, streamName) + require.NoError(t, ferr) + sources := st.CachedInfo().Config.Sources + require.Len(t, sources, 1) + assert.Equal(t, streamNameFor("events.noupdate.parent"), sources[0].Name) + require.Len(t, sources[0].SubjectTransforms, 1) + assert.Equal(t, publishSubjectFor("events.noupdate.parent", "routed.key"), + sources[0].SubjectTransforms[0].Source) + + // Re-asserting that same binding is now itself a no-op. + bd.knownStreams.Delete(streamName) + _, eerr = p.ensureStreamFor(ctx, bd, child) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + t.Fatal("re-declaring an existing binding must not issue a stream update") + case <-time.After(500 * time.Millisecond): + } + + // A SECOND binding onto a parent already in the source set must still + // write. This is the case the skip gets wrong if the config being built + // shares its StreamSource objects with the snapshot it is compared + // against: merging appends to the transforms in place, so both sides move + // together and a genuinely new binding reads as no change at all. A first + // binding hides that — it appends a whole element, so the lengths differ + // regardless. + sibling := &pb.Address{ + Name: "events.noupdate", + Type: pb.Address_TOPIC, + Subjects: []string{"second.key"}, + ParentAddress: &pb.Address{Name: "events.noupdate.parent", Type: pb.Address_TOPIC}, + } + _, eerr = p.ensureStreamFor(ctx, bd, sibling) + require.NoError(t, eerr) + require.NoError(t, bd.nc.Flush()) + select { + case <-updates: + case <-time.After(2 * time.Second): + t.Fatal("a second binding onto an already-bound parent must still update the stream") + } + + st, ferr = bd.js.Stream(ctx, streamName) + require.NoError(t, ferr) + sources = st.CachedInfo().Config.Sources + require.Len(t, sources, 1, "both bindings share one parent, so one source carries both") + assert.Len(t, sources[0].SubjectTransforms, 2, "the earlier binding must survive the later one") +} + +// TestStreamConfigSatisfied covers the comparison that lets ensureStreamFor +// skip an assertion which would change nothing (see +// TestBareStreamAssertionIssuesNoUpdate for why that matters). It must answer +// "satisfied" only when every field the connector manages already matches — +// a false positive there silently drops a real config change. +func TestStreamConfigSatisfied(t *testing.T) { + base := func() *jetstream.StreamConfig { + return &jetstream.StreamConfig{ + Name: "arke_events_x", + Subjects: []string{"events.x.~", "events.x.~.>"}, + Storage: jetstream.FileStorage, + Retention: jetstream.LimitsPolicy, + Duplicates: defaultDedupWindow, + Replicas: 1, + MaxAge: defaultStreamMaxAge, + } + } + assert.True(t, streamConfigSatisfied(base(), base())) + + // Subject order is the server's to choose, not a difference. + reordered := base() + reordered.Subjects = []string{"events.x.~.>", "events.x.~"} + assert.True(t, streamConfigSatisfied(reordered, base())) + + // Each managed field, differing. + for name, mutate := range map[string]func(*jetstream.StreamConfig){ + "subjects": func(c *jetstream.StreamConfig) { c.Subjects = []string{"events.x.~"} }, + "max age": func(c *jetstream.StreamConfig) { c.MaxAge = time.Hour }, + "max bytes": func(c *jetstream.StreamConfig) { c.MaxBytes = 1 << 20 }, + "replicas": func(c *jetstream.StreamConfig) { c.Replicas = 3 }, + "duplicates": func(c *jetstream.StreamConfig) { c.Duplicates = time.Minute }, + "storage": func(c *jetstream.StreamConfig) { c.Storage = jetstream.MemoryStorage }, + "retention": func(c *jetstream.StreamConfig) { c.Retention = jetstream.WorkQueuePolicy }, + } { + t.Run(name+" differs", func(t *testing.T) { + live := base() + mutate(live) + assert.False(t, streamConfigSatisfied(live, base()), + "a differing %s must not read as satisfied", name) + }) + } + + // A stream with no bindings does not satisfy a config that declares one... + want := base() + want.Sources = []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "events.p.~.k", Destination: "events.p.~.k"}}}} + assert.False(t, streamConfigSatisfied(base(), want)) + + // ...and one that already carries exactly that binding does. + live := base() + live.Sources = []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "events.p.~.k", Destination: "events.p.~.k"}}}} + assert.True(t, streamConfigSatisfied(live, want)) + + // An extra binding on the live stream is a difference in the other + // direction: the merged config would have carried it forward, so a + // mismatch here means something else changed it concurrently. + extra := base() + extra.Sources = append(copySources(live.Sources), &jetstream.StreamSource{Name: "arke_events_q"}) + assert.False(t, streamConfigSatisfied(extra, want)) +} + +// TestCopySourcesIsDeep guards the trap that makes the skip logic safe: +// mergeBoundSources appends to a StreamSource's transforms in place, so the +// snapshot ensureStreamFor compares against must not share those objects with +// the config it merges into — or every merge would compare equal to itself and +// a newly declared binding would be skipped instead of written. +func TestCopySourcesIsDeep(t *testing.T) { + orig := []*jetstream.StreamSource{{Name: "arke_events_p", + SubjectTransforms: []jetstream.SubjectTransformConfig{{Source: "a", Destination: "a"}}}} + snapshot := copySources(orig) + + merged := mergeBoundSources(orig, "arke_events_p", []string{"b"}) + require.Len(t, merged[0].SubjectTransforms, 2) + assert.Len(t, snapshot[0].SubjectTransforms, 1, + "the snapshot must not see a transform appended after it was taken") + assert.False(t, sameSources(snapshot, merged), + "a merge that added a binding must read as a difference") +} + +// TestStreamConfigSatisfiedUnlimitedSentinel: JetStream normalizes an unset +// size limit to its -1 "unlimited" sentinel and echoes that back, so a config +// asking for 0 describes the same stream. Comparing the two literally made +// every assertion look like a change, defeating the skip entirely. +func TestStreamConfigSatisfiedUnlimitedSentinel(t *testing.T) { + live := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: -1} + want := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: 0} + assert.True(t, streamConfigSatisfied(live, want), + "an unset max-bytes and the server's -1 sentinel describe the same stream") + + // A real cap is still a difference in both directions. + capped := &jetstream.StreamConfig{Subjects: []string{"a"}, Replicas: 1, MaxBytes: 1 << 20} + assert.False(t, streamConfigSatisfied(live, capped)) + assert.False(t, streamConfigSatisfied(capped, want)) +} From 2ed5296ffdc0d6c414d2b1f612ef40b7cd10a7c8 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 15:09:38 -0400 Subject: [PATCH 61/77] fix: confine the publish-id contract to the unary path amqp091 reads PublishId and PublisherName in exactly one place: publishOneStream (amqp091.go:1544/1638), which only PublishOne reaches, and only for an Address_STREAM. The streaming Publish calls prepareAndSend directly for every address type -- the plain channel publish, which reads neither field. The connector keyed both halves of the contract on the address type alone, so the streaming path inherited rules that belong to the unary one: - a STREAM address was deduplicated on the streaming path, silently discarding a message RabbitMQ stores. A publisher reusing an id there is doing something RabbitMQ told it was free. - a missing PublisherName was refused on every address type, so a streaming publish that works on RabbitMQ errored here. Gate both on the unary path; the unary behaviour is unchanged. TestStreamingPublishIgnoresPublishID now covers STREAM alongside TOPIC and QUEUE -- its old name asserted the wrong scope -- and TestStreamingPublishAcceptsPublishIDWithoutPublisherName pins the refusal to PublishOne. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 30 ++++-- .../provider/connectors/natsjs/natsjs_test.go | 102 +++++++++++++----- 2 files changed, 97 insertions(+), 35 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index a6e1bd0..4c64e96 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -968,7 +968,13 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, // fails outright on RabbitMQ. return &pb.Error{Message: queueDedupError} } - if msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { + // The companion refusal, and unary-only for the same reason: it lives in + // amqp091's publishOneStream (amqp091.go:1544), which only PublishOne + // reaches. The streaming Publish calls prepareAndSend directly and reads + // neither field, so a streaming publish carrying an id but no publisher + // name is accepted on RabbitMQ — refusing it here would break a publisher + // that works there today. + if unary && msg.GetPublishId() > 0 && msg.GetPublisherName() == "" { return &pb.Error{Message: "PublisherName not set on message, PublisherName is required when PublishID is set"} } // A STREAM address must name a stream that already exists: amqp091 routes @@ -998,13 +1004,21 @@ func (p *natsjsProvider) publishMsg(ctx context.Context, bd *natsBrokerDetails, } var pubOpts []jetstream.PublishOpt // publish_id + publisher_name -> Nats-Msg-Id (dedup within the stream window), - // but only where amqp091 would also have deduplicated. On a non-STREAM - // address the id is ignored rather than honoured: amqp091's streaming - // Publish hands every non-STREAM address to the plain channel publish, - // which drops the id on the floor, so applying it here would silently give - // the client a guarantee that disappears on RabbitMQ. (The unary path never - // gets this far with that combination — it is refused above.) - if msg.GetPublishId() > 0 && dedupAddress { + // but only where amqp091 would also have deduplicated: the unary path, on a + // STREAM address. That is the one combination RabbitMQ deduplicates on, + // because publishOneStream is the only place amqp091 forwards the id + // (streamPrepareAndSend, amqp091.go:1638) and PublishOne is the only caller + // that routes there. + // + // Everywhere else the id is ignored rather than honoured, and the streaming + // path is ignored for EVERY address type, STREAM included: amqp091's + // Publish hands them all to the plain channel publish, which drops the id + // on the floor. Deduplicating a STREAM address here because the type looks + // right would silently discard a message RabbitMQ stores — the client + // reusing an id on that path is doing something RabbitMQ told it was free. + // (The unary path never reaches here on a non-STREAM address; it is refused + // above.) + if unary && msg.GetPublishId() > 0 && dedupAddress { pubOpts = append(pubOpts, jetstream.WithMsgID(fmt.Sprintf("%s-%d", msg.GetPublisherName(), msg.GetPublishId()))) } _, perr := bd.js.PublishMsg(ctx, nmsg, pubOpts...) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 6a46ca1..fea71b2 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3049,41 +3049,89 @@ func TestPublishDedupRefusedOnEveryNonStreamAddress(t *testing.T) { PublisherName: "pub", PublishId: 1, Body: []byte("x")})) } -// TestStreamingPublishIgnoresPublishIDOnNonStreamAddress pins the other half of -// the dedup contract: the refusal belongs to the unary path only. amqp091's -// streaming Publish hands every non-STREAM address to the plain channel -// publish, which neither refuses the id nor deduplicates on it — so natsjs -// must not refuse it either (that would be stricter than RabbitMQ, breaking a -// publisher that works there today), and must not honour it either (that would -// be a guarantee RabbitMQ does not give). -func TestStreamingPublishIgnoresPublishIDOnNonStreamAddress(t *testing.T) { +// TestStreamingPublishIgnoresPublishID pins the other half of the dedup +// contract: the whole publish-id contract belongs to the unary path only. +// amqp091's streaming Publish calls prepareAndSend directly for EVERY address +// type — the plain channel publish, which reads neither PublishId nor +// PublisherName — so natsjs must not refuse the id there (that would be +// stricter than RabbitMQ, breaking a publisher that works there today) and +// must not honour it there either (that would be a guarantee RabbitMQ does not +// give, and honouring it discards a message RabbitMQ stores). +// +// Regression: the address type alone decided this, so a STREAM address was +// deduplicated on the streaming path too — silently dropping the second +// publish, where amqp091 never reaches the Streams client that deduplicates. +func TestStreamingPublishIgnoresPublishID(t *testing.T) { + for _, tc := range []struct { + name string + typ pb.Address_TargetType + }{ + {"topic (the proto zero value)", pb.Address_TOPIC}, + {"queue", pb.Address_QUEUE}, + // The one that regressed: STREAM is the type the UNARY path + // deduplicates on, but the streaming path never routes to the client + // that does the deduplicating, so the id has to be dropped here too. + {"stream", pb.Address_STREAM}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + in := make(chan *pb.Message, 2) + errChan := make(chan *pb.Error, 2) + pubCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Publish(pubCtx, in, errChan) + + addr := &pb.Address{Name: "events.stream.dedupignored", Type: tc.typ, Subjects: []string{"k"}} + for i := 0; i < 2; i++ { + in <- &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} + select { + case e := <-errChan: + require.Nil(t, e, "the streaming path must not refuse a publish id: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") + } + } + + stats := p.SourceStats(ctx, &pb.Source{ + Name: "events.stream.dedupignored.consumer", + Address: &pb.Address{Name: "events.stream.dedupignored"}, + }) + assert.Nil(t, stats.GetError()) + assert.Equal(t, int64(2), stats.GetMessageCount(), + "the id must be ignored on the streaming path, not deduplicated on — RabbitMQ drops it on the floor") + }) + } +} + +// TestStreamingPublishAcceptsPublishIDWithoutPublisherName pins the companion +// refusal to the unary path as well. amqp091 requires a PublisherName +// alongside a PublishID only in publishOneStream (amqp091.go:1544), where the +// RabbitMQ Streams client needs a named publisher to deduplicate against. The +// streaming Publish never routes there and reads neither field, so this +// publish succeeds on RabbitMQ and must succeed here. +func TestStreamingPublishAcceptsPublishIDWithoutPublisherName(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) - in := make(chan *pb.Message, 2) - errChan := make(chan *pb.Error, 2) + in := make(chan *pb.Message, 1) + errChan := make(chan *pb.Error, 1) pubCtx, cancel := context.WithCancel(ctx) defer cancel() go p.Publish(pubCtx, in, errChan) - addr := &pb.Address{Name: "events.stream.dedupignored", Type: pb.Address_TOPIC, Subjects: []string{"k"}} - for i := 0; i < 2; i++ { - in <- &pb.Message{Address: addr, Body: []byte("x"), PublisherName: "pub", PublishId: 42} - select { - case e := <-errChan: - require.Nil(t, e, "the streaming path must not refuse a publish id: %v", e.GetMessage()) - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for the publish reply") - } + in <- &pb.Message{ + Address: &pb.Address{Name: "events.stream.noPubName", Type: pb.Address_TOPIC, Subjects: []string{"k"}}, + Body: []byte("x"), + PublishId: 42, + } + select { + case e := <-errChan: + require.Nil(t, e, "the streaming path reads neither PublishId nor PublisherName: %v", e.GetMessage()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the publish reply") } - - stats := p.SourceStats(ctx, &pb.Source{ - Name: "events.stream.dedupignored.consumer", - Address: &pb.Address{Name: "events.stream.dedupignored"}, - }) - assert.Nil(t, stats.GetError()) - assert.Equal(t, int64(2), stats.GetMessageCount(), - "the id must be ignored on a non-STREAM address, not deduplicated on — RabbitMQ drops it on the floor") } // TestHeadersAddressIgnoresRoutingKey covers a headers address that also From 00c874d9199a18baa14463eb339f6ef810ef38ac Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 15:10:50 -0400 Subject: [PATCH 62/77] fix: keep the routing key when dead-lettering a message routed in from a parent An address-to-address binding sources the parent's messages into the child's stream keeping the subject they were published under, and the child's consumer selects exactly those subjects -- so a message the source consumes can be rooted at the PARENT address while the source names the child. routingKeyFromSubject recovered the key by stripping the consuming address's own prefix, which a parent-rooted subject never matches, so it reported an empty routing key for every such message. Dead-lettering one then published it to the bare DLA subject instead of under its original key, where a DLQ consumer bound by routing key would never see it. RabbitMQ preserves the key through both the exchange-to-exchange route and the DLX move. Locate the reserved delimiter instead of stripping a particular address's prefix. That is unambiguous: escapeToken never emits a bare "~" token (an escaped token is "~e", or '~' followed by two-character codes, so at least three characters) and subjectPrefix inserts exactly one, so the first token equal to the delimiter is always the one the connector put there. Signed-off-by: Michael Otteni --- .../provider/connectors/natsjs/helpers.go | 46 +++++++++++----- .../connectors/natsjs/helpers_test.go | 29 +++++++--- internal/provider/connectors/natsjs/natsjs.go | 2 +- .../provider/connectors/natsjs/natsjs_test.go | 53 +++++++++++++++++++ 4 files changed, 107 insertions(+), 23 deletions(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 227ed29..87d7378 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -4,6 +4,7 @@ package natsjs import ( + "slices" "strings" "github.com/nats-io/nats.go" @@ -307,23 +308,40 @@ func translateWildcards(key string) string { } // routingKeyFromSubject recovers the routing key a delivered message was -// published with by stripping the address's subject prefix and decoding each -// escaped token — the inverse of publishSubjectFor for subjects the -// connector itself produced (the decode is what keeps a dead-letter -// republish of the recovered key on the same tokens instead of escaping -// them a second time). The bare prefix (an empty routing key) and any -// subject not under the prefix (which a consumer's filter subjects make -// impossible) yield "". -func routingKeyFromSubject(addressName, subject string) string { - rk, ok := strings.CutPrefix(subject, subjectPrefix(addressName)+".") - if !ok { +// published with by dropping everything up to and including the address +// delimiter and decoding each escaped token — the inverse of publishSubjectFor +// for subjects the connector itself produced (the decode is what keeps a +// dead-letter republish of the recovered key on the same tokens instead of +// escaping them a second time). A subject that is just a prefix (an empty +// routing key) yields "", as does one carrying no delimiter at all. +// +// The delimiter is located rather than a specific address's prefix stripped, +// because a message can legitimately arrive under a subject rooted at a +// DIFFERENT address than the one the source names: an address-to-address +// binding sources the parent's messages into the child's stream keeping the +// subject they were published under (see ensureStreamFor), and the child's +// consumer selects exactly those subjects (filterSubjectsFor). Stripping the +// child's prefix would fail to match and report an empty routing key for every +// message routed in from a parent — dead-lettering it under the wrong key, +// where RabbitMQ's broker-side DLX move preserves the original. +// +// Scanning for the delimiter is unambiguous: escapeToken never emits a bare +// "~" token (an escaped token is "~e", or '~' followed by one or more +// two-character codes, so at least three characters), and subjectPrefix +// inserts exactly one. So the first token equal to the delimiter is always the +// one the connector put there. +func routingKeyFromSubject(subject string) string { + tokens := strings.Split(subject, ".") + delim := slices.Index(tokens, addressDelim) + if delim < 0 || delim == len(tokens)-1 { return "" } - tokens := strings.Split(rk, ".") - for i, t := range tokens { - tokens[i] = unescapeToken(t) + rk := tokens[delim+1:] + out := make([]string, 0, len(rk)) + for _, t := range rk { + out = append(out, unescapeToken(t)) } - return strings.Join(tokens, ".") + return strings.Join(out, ".") } // streamSubjectsFor returns the subject set a stream captures for the given diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 456e31a..ecb757b 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -157,19 +157,32 @@ func TestPublishSubjectFor(t *testing.T) { func TestRoutingKeyFromSubject(t *testing.T) { // inverse of publishSubjectFor assert.Equal(t, "region.us.created", - routingKeyFromSubject("events.orders", "events.orders.~.region.us.created")) + routingKeyFromSubject("events.orders.~.region.us.created")) // the bare prefix is the empty routing key - assert.Equal(t, "", routingKeyFromSubject("events.orders", "events.orders.~")) - // a subject outside the address's space yields no key - assert.Equal(t, "", routingKeyFromSubject("events.orders", "events.audit.~.x")) + assert.Equal(t, "", routingKeyFromSubject("events.orders.~")) + // a subject with no delimiter at all is not one the connector produced + assert.Equal(t, "", routingKeyFromSubject("events.orders")) // empty address: subjects live directly under the delimiter - assert.Equal(t, "a.b", routingKeyFromSubject("", "~.a.b")) + assert.Equal(t, "a.b", routingKeyFromSubject("~.a.b")) + // A message routed in from a parent address keeps the subject it was + // published under, so it is rooted at the PARENT while the source that + // consumes it names the child. The key still has to come back: RabbitMQ's + // DLX move preserves the original routing key, and a dead-letter here is + // the proxy-side stand-in for that move. + // + // Regression: this stripped the consuming address's own prefix, which a + // parent-rooted subject never matches, so every message routed in from a + // parent dead-lettered under an empty key. + assert.Equal(t, "region.us.created", + routingKeyFromSubject(publishSubjectFor("events.parent", "region.us.created"))) // escaped tokens decode back to the published key, so a dead-letter // republish of the recovered key lands on the same tokens (round trip: - // rk -> subject -> rk for keys with reserved characters) - for _, rk := range []string{"a b.c", "x.~.y", "#", "a*b", "region.us"} { + // rk -> subject -> rk for keys with reserved characters). The delimiter + // scan must survive a key that itself contains a literal '~', which + // escapes to "~~" and so is never mistaken for the delimiter token. + for _, rk := range []string{"a b.c", "x.~.y", "#", "a*b", "region.us", "~"} { subj := publishSubjectFor("events.orders", rk) - assert.Equal(t, rk, routingKeyFromSubject("events.orders", subj), "via %q", subj) + assert.Equal(t, rk, routingKeyFromSubject(subj), "via %q", subj) } } diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 4c64e96..0026dd3 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1846,7 +1846,7 @@ func (p *natsjsProvider) DeadLetter(ctx context.Context, source *pb.Source, uuid // message was originally headed. dlSubject := opts["DeadLetterSubject"] if dlSubject == "" { - dlSubject = routingKeyFromSubject(source.GetAddress().GetName(), m.Subject()) + dlSubject = routingKeyFromSubject(m.Subject()) } dlqMsg := &nats.Msg{ Subject: publishSubjectFor(dla, dlSubject), diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index fea71b2..e271659 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -379,6 +379,59 @@ func TestDeadLetterPreservesRoutingKey(t *testing.T) { "the DLQ copy must carry the original routing key under the DLA root") } +// TestDeadLetterPreservesRoutingKeyRoutedFromParent is the same contract for a +// message that reached the source through an address-to-address binding. +// Sourcing keeps the subject the message was published under, so it stays +// rooted at the PARENT address while the source names the child — and RabbitMQ +// preserves the original routing key through both the exchange-to-exchange +// route and the DLX move, so the DLQ copy must still carry it. +// +// Regression: the routing key was recovered by stripping the consuming +// address's own prefix, which a parent-rooted subject never matches, so every +// message routed in from a parent dead-lettered under an empty key — landing +// on the bare DLA subject where a DLQ consumer bound by routing key would +// never see it. +func TestDeadLetterPreservesRoutingKeyRoutedFromParent(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + parent := &pb.Address{Name: "events.dlrk.parent", Type: pb.Address_TOPIC} + child := &pb.Address{ + Name: "events.dlrk.child", + Type: pb.Address_TOPIC, + Subjects: []string{"region.us.created"}, + ParentAddress: parent, + } + + out := make(chan *pb.Message, 1) + src := &pb.Source{Name: "events.dlrk.child.consumer", Type: pb.Source_QUEUE, Address: child, + Options: map[string]string{ + "Offset": "first", + "DeadLetterAddress": "events.dlrk.routed.dlq", + }} + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + time.Sleep(500 * time.Millisecond) + + // Published to the PARENT: the binding sources it into the child's stream + // under its original, parent-rooted subject. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.dlrk.parent", Subjects: []string{"region.us.created"}}, + Body: []byte("poison")})) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + dlqStream, serr := bd.js.Stream(ctx, streamNameFor("events.dlrk.routed.dlq")) + require.NoError(t, serr) + raw, gerr := dlqStream.GetMsg(ctx, 1) + require.NoError(t, gerr) + assert.Equal(t, "events.dlrk.routed.dlq.~.region.us.created", raw.Subject, + "a message routed in from a parent must dead-letter under its original routing key") +} + // TestDeadLetterFailureKeepsMessage: when the dead-letter publish cannot happen // (here the DLQ address's subject space is already claimed by a foreign // stream, so ensuring its stream fails), DeadLetter must NOT Term the From f73fdfb5506e2fce1eab1a0ef28d993eca7939e6 Mon Sep 17 00:00:00 2001 From: Richard Sugg Date: Tue, 21 Jul 2026 15:52:25 -0400 Subject: [PATCH 63/77] fix: add timestamp_in_ms header Signed-off-by: Richard Sugg --- .../provider/connectors/natsjs/helpers.go | 31 ++++++++++++++----- internal/provider/connectors/natsjs/natsjs.go | 2 +- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 9439086..b980fc1 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -4,10 +4,14 @@ package natsjs import ( + "strconv" "strings" + "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/internal/util" ) // Subject / topology mapping. @@ -77,6 +81,8 @@ var tokenUnescapes = map[byte]byte{ 'a': '*', 'g': '>', 'h': '#', } +const HeaderNatsTimeStamp = "Nats-Time-Stamp" // Header key for NATS timestamp in milliseconds + // tokenEscapeTriggers are the characters that force a token into the escaped // form; a token free of them passes through literally. const tokenEscapeTriggers = "~ \t\r\n\f*>#" //nolint:gosec // G101 false positive: NATS subject reserved characters, not a credential @@ -633,18 +639,22 @@ func copyHeader(in nats.Header) nats.Header { return out } -// natsToPbHeader converts a NATS header to Arke's flat string headers, keeping +// natsToPbHeader creates Arke's flat string headers, keeping // the first value of each key. -func natsToPbHeader(in nats.Header) map[string]string { - if len(in) == 0 { - return map[string]string{} - } - out := make(map[string]string, len(in)) - for k, vals := range in { +func natsToPbHeader(m jetstream.Msg) map[string]string { + hdrs := m.Headers() + out := make(map[string]string, len(hdrs)) + for k, vals := range hdrs { if len(vals) > 0 { out[k] = vals[0] } } + meta, err := m.Metadata() + if err != nil { + util.Logger.Debugf("natsToPbHeader: failed to get metadata: %v", err) + } else { + out["timestamp_in_ms"] = strconv.FormatInt(meta.Timestamp.UnixNano()/int64(time.Millisecond), 10) + } return out } @@ -684,3 +694,10 @@ func filterMatches(f *pb.Filter, headers map[string]string) bool { // ALL: every match passed; ANY: none passed. return !any } + +func DumpMsg(m jetstream.Msg) { + util.Logger.Debugf("DumpMsg: subject=%s", m.Subject()) + util.Logger.Debugf("DumpMsg: reply=%s", m.Reply()) + util.Logger.Debugf("DumpMsg: len(data)=%d", len(m.Data())) + util.Logger.Debugf("DumpMsg: headers=%+v", m.Headers()) +} diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 25cddfd..e453d34 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1385,7 +1385,7 @@ func isStartPositionConflict(err error) bool { func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetails, source *pb.Source, subKey string, m jetstream.Msg, out chan<- *pb.Message) { defer util.RecoverPanic() - headers := natsToPbHeader(m.Headers()) + headers := natsToPbHeader(m) // Synthesize x-retry-count from JetStream's delivery count so a client's retry // policy works without RabbitMQ's x-death header. if md, err := m.Metadata(); err == nil && md.NumDelivered > 1 { From badafb31132651a37c8f3978cacd5b2e44362280 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 15:58:46 -0400 Subject: [PATCH 64/77] fix: stop the NATS wire format silently dropping headers nats.go serializes headers with net/http's Header.Write (Msg.headerBytes). That writer is lossy twice, and both losses are silent -- no error ever reaches the publisher: - a name failing httpguts.ValidHeaderFieldName is dropped outright. The stdlib says so in a comment: "we have no good way to provide the error back ... so just drop invalid headers instead". - values get CR/LF replaced with spaces, then leading and trailing space and tab trimmed (textproto.TrimString). AMQP field-table keys are length-prefixed binary, so RabbitMQ round-trips a header name containing spaces, colons or non-ASCII exactly. Measured against a live broker of each kind through one proxy: of 38 header names built from punctuation and non-ASCII, amqp091 delivered all 38 and natsjs delivered 15. Values with surrounding whitespace arrived trimmed on natsjs and intact on amqp091 -- that half needs no exotic name at all, so it can corrupt an entirely conventional header. Escape any entry NATS cannot carry verbatim, the way escapeToken already does for subjects: the mapping is injective and conventional names pass through untouched, so ordinary traffic stays natively readable to a non-Arke consumer and only the unrepresentable entries are encoded. Encoding name and value together keeps one rule and one decode path. Unpadded base64url is used because its alphabet is entirely valid header-name characters. A name already carrying the escape prefix is escaped too, which is what makes the mapping injective: an unescaped name on the wire provably never starts with the prefix, so decoding only ever fires on entries this connector encoded. The header round trip had no test coverage at all. It does now, including one that drives the escaped form through the actual http.Header.Write rather than a restatement of its rule, so stdlib drift fails the test rather than silently resurrecting the bug. Signed-off-by: Michael Otteni --- .../provider/connectors/natsjs/helpers.go | 103 +++++++++++++- .../connectors/natsjs/helpers_test.go | 126 ++++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 52 ++++++++ 3 files changed, 276 insertions(+), 5 deletions(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 87d7378..07ac86b 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -4,6 +4,7 @@ package natsjs import ( + "encoding/base64" "slices" "strings" @@ -645,18 +646,96 @@ func wantsStoredPosition(source *pb.Source) bool { return strings.EqualFold(strings.TrimSpace(source.GetOptions()["Offset"]), "continue") } -// pbToNatsHeader converts Arke's flat string headers to a NATS header. +// Header mapping. +// +// AMQP field-table keys are length-prefixed binary: any bytes are legal, and +// RabbitMQ round-trips a header name containing spaces, colons or non-ASCII +// exactly. NATS headers are HTTP/MIME text lines, and nats.go serializes them +// with net/http's Header.Write (nats.go Msg.headerBytes -> http.Header.Write). +// That writer is lossy in two ways, both SILENT: +// +// name fails httpguts.ValidHeaderFieldName -> the entry is dropped outright +// ("we have no good way to provide the error back ... so just drop +// invalid headers instead", net/http/header.go) +// value -> CR/LF replaced with spaces, then leading/trailing space and tab +// trimmed (textproto.TrimString) +// +// So a straight pass-through loses headers RabbitMQ delivers, with no error to +// the publisher. Measured against a live broker of each kind: of 38 header +// names built from punctuation and non-ASCII, amqp091 delivered all 38 and an +// unescaped natsjs delivered 15. +// +// We therefore escape any entry NATS cannot carry verbatim, exactly as +// escapeToken does for subjects: the mapping is injective and conventional +// names pass through untouched, so ordinary traffic stays natively readable by +// non-Arke NATS consumers. An escaped entry becomes +// +// "": "" +// +// Encoding both halves together keeps one rule and one decode path rather than +// a separate marker for value-only damage. base64url without padding is used +// because its alphabet (A-Za-z0-9, '-', '_') is entirely valid header-name +// characters; standard base64's '+', '/' and '=' are not. +// +// A name that already carries the prefix is escaped too, even when it would +// survive as-is. That is what makes the mapping injective: an unescaped name on +// the wire provably never starts with the prefix, so decoding only ever fires +// on entries the connector itself encoded, and a client may legitimately send a +// header named like an encoded one without it being decoded into something else. +const headerEscapePrefix = "X-Arke-Esc-" + +// tchar reports whether c is valid in an HTTP header field name (RFC 7230 +// token). Checked inline rather than via httpguts to keep the rule explicit +// next to the escaping it drives, and to avoid a dependency for six lines. +func tchar(c byte) bool { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + return true + } + return strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0 +} + +// headerNameSurvives reports whether a name reaches the broker unchanged. +func headerNameSurvives(name string) bool { + if name == "" || strings.HasPrefix(name, headerEscapePrefix) { + return false + } + for i := 0; i < len(name); i++ { + if !tchar(name[i]) { + return false + } + } + return true +} + +// headerValueSurvives reports whether a value reaches the broker unchanged, +// mirroring what http.Header.Write does to it: CR/LF become spaces, then +// leading and trailing space/tab are trimmed. +func headerValueSurvives(value string) bool { + return !strings.ContainsAny(value, "\r\n") && value == strings.Trim(value, " \t") +} + +// pbToNatsHeader converts Arke's flat string headers to a NATS header, +// escaping any entry the NATS wire format would drop or rewrite. func pbToNatsHeader(in map[string]string) nats.Header { if len(in) == 0 { return nil } h := nats.Header{} for k, v := range in { - h.Set(k, v) + if headerNameSurvives(k) && headerValueSurvives(v) { + h.Set(k, v) + continue + } + h.Set(headerEscapePrefix+b64.EncodeToString([]byte(k)), b64.EncodeToString([]byte(v))) } return h } +// b64 is unpadded base64url: its alphabet is entirely valid header-name +// characters, so an encoded name needs no further escaping. +var b64 = base64.RawURLEncoding + // copyHeader shallow-copies a NATS header so options applied to the copy at // publish time (e.g. a message id) do not mutate the source message's header // map. @@ -672,16 +751,30 @@ func copyHeader(in nats.Header) nats.Header { } // natsToPbHeader converts a NATS header to Arke's flat string headers, keeping -// the first value of each key. +// the first value of each key and restoring any entry pbToNatsHeader escaped. +// +// An entry carrying the escape prefix whose name or value does not decode is +// passed through verbatim rather than dropped: the connector never produces +// such an entry, so it can only come from a foreign publisher, and surfacing it +// unchanged loses nothing. func natsToPbHeader(in nats.Header) map[string]string { if len(in) == 0 { return map[string]string{} } out := make(map[string]string, len(in)) for k, vals := range in { - if len(vals) > 0 { - out[k] = vals[0] + if len(vals) == 0 { + continue + } + name, value := k, vals[0] + if encoded, ok := strings.CutPrefix(k, headerEscapePrefix); ok { + dn, nerr := b64.DecodeString(encoded) + dv, verr := b64.DecodeString(value) + if nerr == nil && verr == nil { + name, value = string(dn), string(dv) + } } + out[name] = value } return out } diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index ecb757b..439f2b6 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -4,11 +4,15 @@ package natsjs import ( + "bytes" + "net/http" "strings" "testing" + "github.com/nats-io/nats.go" pb "github.com/sassoftware/arke/api" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStreamNameFor(t *testing.T) { @@ -497,3 +501,125 @@ func TestParentBindingSubjectsHeadersParent(t *testing.T) { } assert.Equal(t, []string{"events.parent.~", "events.parent.~.>"}, parentBindingSubjects(addr)) } + +// TestHeaderRoundTripPreservesEverything: the NATS wire format silently drops +// header names that are not HTTP tokens and trims whitespace off values (see +// the header-mapping comment in helpers.go). Every one of these round-trips +// through a real RabbitMQ unchanged, so the escaping has to make them survive +// here too. The punctuation set is the one measured against a live broker of +// each kind, where an unescaped connector delivered 15 of the 38 names. +func TestHeaderRoundTripPreservesEverything(t *testing.T) { + in := map[string]string{ + // conventional headers — must pass through unescaped + "Content-Type": "application/vnd.example.event+json", + "__namespace": "tenant-a", + "x-event-address": "events.orders", + "traceparent": "00-abc-def-01", + "X-B3-TraceId": "abc", + "x-retry-count": "3", + // values the writer would rewrite + "empty-value": "", + "leading-space": " v", + "trailing-space": "v ", + "surrounding-tab": "\tv\t", + "embedded-newline": "a\nb", + "embedded-crlf": "a\r\nb", + "only-whitespace": " ", + "internal-space-ok": "a b c", + "unicode-value": "héllo wörld", + "looks-like-escaped": headerEscapePrefix + "Zm9v", + } + // names built from every character the live probe exercised + for _, c := range "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \tàé€中" { + in[string([]rune{'h', 'd', 'r', c, 'k', 'e', 'y'})] = "v" + } + + got := natsToPbHeader(pbToNatsHeader(in)) + assert.Equal(t, in, got, "every header must survive the round trip byte-for-byte") +} + +// TestHeaderEscapingOnlyWhereNeeded: conventional names and values reach the +// broker verbatim, so ordinary traffic stays readable to a native NATS +// consumer and only the entries NATS cannot carry are encoded. +func TestHeaderEscapingOnlyWhereNeeded(t *testing.T) { + h := pbToNatsHeader(map[string]string{ + "Content-Type": "application/json", + "x-event-src": "inventory", + "has space": "v", + "trailing": "v ", + }) + + assert.Equal(t, "application/json", h.Get("Content-Type")) + assert.Equal(t, "inventory", h.Get("x-event-src")) + // the two damaged entries are carried under the escape prefix instead + assert.Empty(t, h.Get("has space")) + assert.Empty(t, h.Get("trailing")) + escaped := 0 + for k := range h { + if strings.HasPrefix(k, headerEscapePrefix) { + escaped++ + } + } + assert.Equal(t, 2, escaped, "exactly the unrepresentable entries are escaped") +} + +// TestHeaderEscapingSurvivesTheRealWriter drives the escaped form through the +// actual lossy component — net/http's Header.Write, which is what nats.go uses +// in Msg.headerBytes — and reads it back the way nats.go parses it. Asserting +// against the real writer rather than against tchar (our restatement of its +// rule) is the point: if the stdlib rule and ours ever diverge, this fails +// while a test written against tchar alone would keep passing. +func TestHeaderEscapingSurvivesTheRealWriter(t *testing.T) { + in := map[string]string{ + "hdr key\twith\r\neverything: €": " spaced\r\nvalue ", + "Content-Type": "application/json", + "plain": "v", + "trailing": "v ", + } + + var buf bytes.Buffer + require.NoError(t, http.Header(pbToNatsHeader(in)).Write(&buf)) + buf.WriteString("\r\n") + + // Parse back preserving original case, as nats.go's readMIMEHeader does. + onWire := nats.Header{} + for _, line := range strings.Split(buf.String(), "\r\n") { + name, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + onWire[name] = append(onWire[name], strings.TrimLeft(value, " \t")) + } + + assert.Equal(t, in, natsToPbHeader(onWire), + "headers must survive the real http.Header.Write round trip") +} + +// TestHeaderDecodeLeavesForeignEntriesAlone: a publisher that is not this +// connector can put anything on a subject, including a name that merely looks +// escaped. Such an entry is passed through rather than dropped or mangled. +func TestHeaderDecodeLeavesForeignEntriesAlone(t *testing.T) { + got := natsToPbHeader(nats.Header{ + headerEscapePrefix + "not!valid!base64": []string{"dg"}, + headerEscapePrefix + "Zm9v": []string{"not!base64"}, + "ordinary": []string{"v"}, + }) + assert.Equal(t, map[string]string{ + headerEscapePrefix + "not!valid!base64": "dg", + headerEscapePrefix + "Zm9v": "not!base64", + "ordinary": "v", + }, got) +} + +// TestHeaderNameSurvives pins the RFC 7230 token rule the escaping keys on. +func TestHeaderNameSurvives(t *testing.T) { + for _, ok := range []string{"Content-Type", "__namespace", "x-retry-count", + "a.b.c", "A1", "!#$%&'*+-.^_`|~"} { + assert.True(t, headerNameSurvives(ok), "%q should pass through", ok) + } + for _, bad := range []string{"", "has space", "has:colon", "has/slash", + "has@at", "has,comma", "has=eq", "has[br]", "unicodeé", + headerEscapePrefix + "x"} { + assert.False(t, headerNameSurvives(bad), "%q must be escaped", bad) + } +} diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index e271659..ad269d9 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3466,3 +3466,55 @@ func TestStreamConfigSatisfiedUnlimitedSentinel(t *testing.T) { assert.False(t, streamConfigSatisfied(live, capped)) assert.False(t, streamConfigSatisfied(capped, want)) } + +// TestHeadersSurviveARealBrokerRoundTrip is the end-to-end half of the header +// escaping: the helper tests prove the encoding is reversible and that it +// survives net/http's writer, but only a real publish and consume proves the +// connector actually routes headers through that encoding on both sides. +// +// Every name here round-trips through RabbitMQ unchanged. Without escaping the +// NATS wire format drops the punctuation and non-ASCII names outright and trims +// the padded values, silently — measured against a live broker of each kind as +// 15 of 38 names delivered. +func TestHeadersSurviveARealBrokerRoundTrip(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + sent := map[string]string{ + "Content-Type": "application/vnd.example.event+json", + "__namespace": "tenant-a", + "x-event-address": "events.orders", + "traceparent": "00-abc-def-01", + "has space": "spacekey", + "has:colon": "colonvalue", + "has/slash": "slashkey", + "unicode-vàlue": "héllo wörld", + "中文": "cjk", + "padded-value": " keep my spaces ", + "tabbed-value": "\tkeep\ttabs\t", + "empty-value": "", + } + + addr := &pb.Address{Name: "events.hdrroundtrip", Subjects: []string{"probe"}} + out := make(chan *pb.Message, 4) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.hdrroundtrip.consumer", Type: pb.Source_QUEUE, Address: addr, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: sent, Body: []byte("probe")})) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + got := m.GetHeaders() + for k, want := range sent { + v, ok := got[k] + assert.True(t, ok, "header %q must be delivered, not silently dropped", k) + assert.Equal(t, want, v, "header %q must arrive byte-for-byte", k) + } +} From 68cf1333b59bdf585e1aa02459d7c0df3ad62ef0 Mon Sep 17 00:00:00 2001 From: Richard Sugg Date: Wed, 22 Jul 2026 10:27:02 -0400 Subject: [PATCH 65/77] test: skip inapplicable test Signed-off-by: Richard Sugg --- tests/integration/integration_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 11d3124..34530c5 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/uuid" pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/test/config" mf "github.com/sassoftware/arke/test/messagefunctions" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -3730,6 +3731,10 @@ func Test_SourceStatsNoStream(t *testing.T) { } } func TestStreamHeaderReceivedTimeEqualsTimestampInMs(t *testing.T) { + config := config.ConnectionConfigurationFromEnv() + if config.GetProvider() != "amqp091" { + t.Skip("Skipping test because it is only applicable to RabbitMQ broker") + } producerConnection := connect() defer producerConnection.Close() expectedMessageCount := 1 From a432ac66dbc0d4d7027ab098409b6c6f538ea788 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 19:31:14 -0400 Subject: [PATCH 66/77] fix: release a single-active pin before clearing its config, not after A durable that drops SingleActiveConsumer has its priority-group config cleared immediately by CreateOrUpdateConsumer (confirmed live: PriorityPolicy=0, PriorityGroups=[] right after the update), but the server's own pin is runtime state independent of that config. A plain (non-priority-group) pull against the downgraded consumer received nothing -- NumWaiting=1, NumPending=1, zero deliveries -- until the previous pin's TTL elapsed (NATSJS_SAC_PINNED_TTL, default 1m), with no error or warning anywhere. The upgrade direction (plain -> SAC) was already tested and works; the reverse had no test in either direction. Worse: once the clearing update has already run, explicitly unpinning the group (Stream.UnpinConsumer, exported since nats-server 2.11) itself fails with "priority group does not exist for this consumer" -- verified against the embedded server before writing any fix. So the release has to happen before the update, not after. releaseStalePins reads the durable's live config first (a plain lookup; ErrConsumerNotFound on an ordinary first-ever subscribe costs nothing) and unpins whatever priority group it still carries, called from Subscribe only when a durable subscribe does not want SingleActiveConsumer. TestSingleActiveConsumerDowngradeReleasesPinPromptly runs at the connector's real default TTL rather than a shortened test-only one, so it proves the wait is gone rather than just faster; proven to fail (5s timeout, no delivery) with the fix's call site stubbed out. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 9 +++- internal/provider/connectors/natsjs/natsjs.go | 35 +++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 51 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 7c03945..a009f24 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -260,7 +260,14 @@ so single-active cannot be enforced; the connector detects that from the effective consumer config and logs a warning that consumers will compete. Priority-group config is update-mutable, so a durable created before its source set `SingleActiveConsumer` is upgraded in place, keeping its name and -ack position. +ack position. The reverse transition needs an extra step: a durable coming +OFF `SingleActiveConsumer` has its pin released explicitly (`UnpinConsumer`) +before the update clears the priority-group config, because unpinning AFTER +the group is gone from the config is rejected by the server ("priority group +does not exist for this consumer"). Skipping that step leaves the previous +pin as server-side state independent of the declared config, silently +blocking ordinary delivery to the downgraded consumer for up to +`NATSJS_SAC_PINNED_TTL` with no error anywhere. When a subscription ends, its delivered-but-unresolved messages are released: their acks could only have arrived on the consume stream that just closed, so diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 5acf1ba..9955739 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1200,6 +1200,20 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c defer bd.releaseDurableFilter(consCfg.Durable) } + // A durable that previously had SingleActiveConsumer pinned a client to + // its priority group. If this subscribe no longer asks for that, the pin + // must be released HERE, before CreateOrUpdateConsumer below clears the + // group from the consumer's declared config: once the group is gone from + // the config, unpinning it errors ("priority group does not exist for + // this consumer" — verified against an embedded server), so a pin + // released only after the update can no longer be released at all. Left + // alone, the stale pin blocks ordinary (non-priority-group) delivery for + // up to NATSJS_SAC_PINNED_TTL (default 1m) with no error or warning + // anywhere, even though the live config already shows no priority policy. + if consCfg.Durable != "" && !source.GetSingleActiveConsumer() { + releaseStalePins(tctx, stream, consCfg.Durable) + } + cons, cerr := stream.CreateOrUpdateConsumer(tctx, consCfg) if cerr != nil && consCfg.Durable != "" && isStartPositionConflict(cerr) { // A durable's start position (DeliverPolicy/OptStartSeq) is fixed at @@ -1620,6 +1634,27 @@ var startPositionConflictMessages = []string{ "start time can not be updated", } +// releaseStalePins unpins every priority group an existing durable's live +// config still carries, ahead of a subscribe that is about to strip +// SingleActiveConsumer's config from it (see the call site in Subscribe). +// Best-effort and read-first: a durable that does not exist yet (the +// ordinary first-subscribe case) answers ErrConsumerNotFound immediately and +// nothing is unpinned; a durable that was never single-active has an empty +// group list and the loop below does nothing. Only a durable coming OFF a +// pin pays the extra UnpinConsumer call. +func releaseStalePins(ctx context.Context, stream jetstream.Stream, durable string) { + existing, err := stream.Consumer(ctx, durable) + if err != nil { + return + } + for _, group := range existing.CachedInfo().Config.PriorityGroups { + if uerr := stream.UnpinConsumer(ctx, durable, group); uerr != nil { + util.Logger.Debugf("natsjs: release stale single-active pin on %s (group %s): %s", + durable, group, uerr.Error()) + } + } +} + // isStartPositionConflict reports whether err is JetStream refusing to move an // existing durable consumer's start position (DeliverPolicy / OptStartSeq / // OptStartTime) — the one configuration conflict Subscribe absorbs by diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 103bfd8..6df6519 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3560,3 +3560,54 @@ func TestTimestampHeaderIsSynthesized(t *testing.T) { assert.GreaterOrEqual(t, ms, before, "must be the broker store time, not the publisher's value") assert.LessOrEqual(t, ms, time.Now().UnixMilli()+1000) } + +// TestSingleActiveConsumerDowngradeReleasesPinPromptly: a durable that drops +// SingleActiveConsumer must resume ordinary (non-pinned) delivery promptly. +// Without releaseStalePins, CreateOrUpdateConsumer clears the priority-group +// config immediately but the server's own pin persists as runtime state +// independent of that config (verified against the embedded server: unpinning +// AFTER the config is cleared errors "priority group does not exist for this +// consumer") — so a plain re-subscribe would silently receive nothing until +// NATSJS_SAC_PINNED_TTL (default 1m) elapsed, with no error anywhere. This +// pins the fix at the connector's default TTL, not a shortened test-only one, +// so it actually proves the wait is gone rather than just faster. +func TestSingleActiveConsumerDowngradeReleasesPinPromptly(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.sacdowngrade", Subjects: []string{"e"}} + sac := sacSource("events.sacdowngrade.consumer", "events.sacdowngrade", "e") + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + errCh := make(chan *pb.Error, 1) + go func() { errCh <- p.Subscribe(subCtx, sac, out) }() + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + cancel() + require.Nil(t, <-errCh) + + // Re-subscribe to the same durable without SingleActiveConsumer. + plain := queueSource("events.sacdowngrade.consumer", "events.sacdowngrade", "e") + out2 := make(chan *pb.Message, 1) + subCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + go p.Subscribe(subCtx2, plain, out2) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + select { + case m := <-out2: + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, "m1", string(m.GetBody())) + case <-time.After(5 * time.Second): + t.Fatal("plain re-subscribe never received a message: the prior single-active pin was not released") + } + + bd := p.getBrokerDetailsByIdentifier("test-client") + stream, serr := bd.js.Stream(ctx, streamNameFor("events.sacdowngrade")) + require.NoError(t, serr) + cons, cerr := stream.Consumer(ctx, durableName(plain)) + require.NoError(t, cerr) + assert.Empty(t, cons.CachedInfo().Config.PriorityGroups, "the downgraded consumer must carry no priority group") +} From 4ecac38e7d61154328c3c70d52ada6e806bcfbe4 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 19:33:05 -0400 Subject: [PATCH 67/77] fix: stamp x-current-offset on STREAM deliveries amqp091's streamSubscribe stamps x-current-offset on every STREAM-source delivery from the RabbitMQ Streams consumer's own offset (amqp091.go:1279); natsjs never carried it, a plain parity gap. The JetStream analogue of that per-reader position is the message's own stream sequence, converted through the existing offsetOf helper -- the same Offset vocabulary SourceStats already reports and accepts, so a value read here and handed back as a source's Offset resumes at the same message. Guarded to STREAM sources only, matching amqp091 exactly: queueSubscribe never sets this header, so a QUEUE/TOPIC source must not carry it either. Both directions covered by TestStreamDeliveryStampsCurrentOffset and TestQueueDeliveryOmitsCurrentOffset; the STREAM test proven to fail (empty header) with the fix's guard disabled. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 1 + internal/provider/connectors/natsjs/natsjs.go | 17 +++++++ .../provider/connectors/natsjs/natsjs_test.go | 51 +++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index a009f24..37fce18 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -649,6 +649,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Max message size | Partial | NATS caps a payload at the server's `max_payload` (1MB default) where RabbitMQ's `max_message_size` default is 16MB (see limitations). | | Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | | RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | +| Stream reader position header (`x-current-offset`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`; the message's own stream sequence in the `Offset` vocabulary. | ## Source options diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 9955739..5830f59 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -53,6 +53,16 @@ const retryCountHeaderName = "x-retry-count" // value is replaced rather than kept. const timestampHeaderName = "timestamp_in_ms" +// currentOffsetHeaderName mirrors the amqp091 connector's streamSubscribe, +// which stamps it on every STREAM-source delivery from the RabbitMQ Streams +// consumer's own offset (amqp091.go:1279). The JetStream analogue of that +// per-reader position is the message's own stream sequence, converted to the +// Offset vocabulary SourceStats already uses (see offsetOf) so a value read +// here and handed back as a source's Offset option resumes at the same +// message. Queue/topic sources never carry this header on either connector — +// amqp091 only sets it on the RabbitMQ-Streams delivery path. +const currentOffsetHeaderName = "x-current-offset" + // Error strings a client may match on, so they have to read identically // whichever connector answers. Both take amqp091's wording: noMessageError is // what it returns for an ack, nack, or dead-letter naming a message it does @@ -1687,6 +1697,13 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai // timestamp_in_ms unconditionally, matching the overwrite = true // RabbitMQ ingress interceptor it stands in for. headers[timestampHeaderName] = strconv.FormatInt(md.Timestamp.UnixMilli(), 10) + // x-current-offset, STREAM sources only, matching amqp091's + // streamSubscribe (queue/topic sources never carry it on either + // connector). The message's own stream sequence is this reader's + // current position in the log at the point of delivery. + if source.GetType() == pb.Source_STREAM { + headers[currentOffsetHeaderName] = strconv.FormatInt(offsetOf(md.Sequence.Stream), 10) + } } // Proxy-side replacement for RabbitMQ headers-exchange routing. A failed diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 6df6519..e3308b0 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3611,3 +3611,54 @@ func TestSingleActiveConsumerDowngradeReleasesPinPromptly(t *testing.T) { require.NoError(t, cerr) assert.Empty(t, cons.CachedInfo().Config.PriorityGroups, "the downgraded consumer must carry no priority group") } + +// TestStreamDeliveryStampsCurrentOffset: amqp091's streamSubscribe stamps +// x-current-offset on every STREAM-source delivery from the RabbitMQ Streams +// consumer's own offset (amqp091.go:1279). The JetStream analogue is the +// message's own stream sequence in the Offset vocabulary SourceStats already +// uses, so a value read here and handed back as a source's Offset resumes at +// the same message. +func TestStreamDeliveryStampsCurrentOffset(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.curoffset", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m0")})) + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m1")})) + + out := make(chan *pb.Message, 4) + src := streamSource("events.curoffset.consumer", "events.curoffset", "first", "e") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m0 := recv(t, out) + require.Nil(t, p.Ack(ctx, m0.GetUuid())) + assert.Equal(t, "0", m0.GetHeaders()[currentOffsetHeaderName], + "the first message in a stream must report offset 0, matching the RabbitMQ-Streams vocabulary") + + m1 := recv(t, out) + require.Nil(t, p.Ack(ctx, m1.GetUuid())) + assert.Equal(t, "1", m1.GetHeaders()[currentOffsetHeaderName]) +} + +// TestQueueDeliveryOmitsCurrentOffset: amqp091 only stamps x-current-offset on +// the RabbitMQ-Streams delivery path (streamSubscribe) — queueSubscribe never +// sets it — so a QUEUE source must not carry it either. +func TestQueueDeliveryOmitsCurrentOffset(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.noqcuroffset", Subjects: []string{"e"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.noqcuroffset.consumer", "events.noqcuroffset", "e"), out) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + _, ok := m.GetHeaders()[currentOffsetHeaderName] + assert.False(t, ok, "x-current-offset is a STREAM-only header on amqp091 too; a queue source must not carry it") +} From da307e8939358a3c4be7f5a030fca1361e06299b Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 19:35:49 -0400 Subject: [PATCH 68/77] fix: decompress gzip STREAM bodies, matching amqp091 amqp091's streamSubscribe decompresses a gzip-tagged body and strips the header (amqp091.go:1286-1292); natsjs passed the compressed bytes straight through -- body corruption, not just a missing header, for whatever consumer read it. Root-caused before porting the fix over blindly: amqp091's OWN STREAM publish path transparently gzips a body that exceeds the RabbitMQ-Streams client library's ~1MiB framing limit (compression.go), a workaround for that library's own ceiling. JetStream's max_payload has no equivalent (deploy-configured to 8MB here), so natsjs correctly never compresses on publish -- it was never going to hit the RabbitMQ problem this exists to solve. But the decompression check is a plain Transfer-Encoding: gzip value match, not tied to who set it, so a publisher that gzips its own body and sets the header itself needs the same treatment on both brokers -- and that publisher-driven case is the one natsjs actually needs to handle. decompressBody mirrors amqp091's own implementation. Guarded to STREAM sources only, matching amqp091 exactly: QUEUE/TOPIC sources pass the header and body through untouched on both connectors, covered by TestQueueGzipBodyPassesThroughUntouched. TestStreamGzipBodyIsDecompressed proven to fail (raw gzip bytes, Transfer-Encoding header still present) with the fix's branch disabled. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 1 + .../provider/connectors/natsjs/helpers.go | 25 +++++++ internal/provider/connectors/natsjs/natsjs.go | 29 +++++++- .../provider/connectors/natsjs/natsjs_test.go | 72 +++++++++++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 37fce18..04d7315 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -650,6 +650,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | | RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | | Stream reader position header (`x-current-offset`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`; the message's own stream sequence in the `Offset` vocabulary. | +| Gzip body passthrough (`Transfer-Encoding: gzip`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`: a gzip-tagged body is decompressed and the header stripped. natsjs never compresses on publish — `max_payload` has no equivalent to the RabbitMQ-Streams client's ~1MiB ceiling amqp091 works around — so this only ever undoes a publisher's own compression. | ## Source options diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 07ac86b..2144fd8 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -4,7 +4,10 @@ package natsjs import ( + "bytes" + "compress/gzip" "encoding/base64" + "io" "slices" "strings" @@ -815,3 +818,25 @@ func filterMatches(f *pb.Filter, headers map[string]string) bool { // ALL: every match passed; ANY: none passed. return !any } + +// decompressBody undoes gzip compression on a STREAM-source message body, +// mirroring amqp091's streamSubscribe (which undoes its own STREAM-publish +// workaround for the RabbitMQ-Streams client's ~1MiB framing limit, but +// applies to any body tagged Transfer-Encoding: gzip regardless of who +// compressed it). natsjs never compresses on publish — JetStream's +// max_payload has no equivalent 1MiB library ceiling — so this only ever +// fires for a publisher's own pre-compressed body. +func decompressBody(b []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + out, err := io.ReadAll(r) + if cerr := r.Close(); cerr != nil && err == nil { + err = cerr + } + if err != nil { + return nil, err + } + return out, nil +} diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 5830f59..1abb38d 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -63,6 +63,18 @@ const timestampHeaderName = "timestamp_in_ms" // amqp091 only sets it on the RabbitMQ-Streams delivery path. const currentOffsetHeaderName = "x-current-offset" +// transferEncodingHeaderName mirrors the amqp091 connector's streamSubscribe. +// amqp091's own STREAM publish path transparently gzips a body over the +// RabbitMQ-Streams client library's ~1MiB framing limit (compression.go) and +// tags it with this header so the delivery path can undo it; JetStream has no +// equivalent library ceiling (max_payload is deploy-configured well above +// 1MiB), so natsjs never compresses on publish. But the header is a plain +// value match, not tied to arke's own compression: a publisher that gzips its +// own body and sets this header itself gets decompressed on amqp091 +// regardless of who set it, so natsjs must honor it too rather than handing a +// STREAM consumer raw gzip bytes it never asked to receive compressed. +const transferEncodingHeaderName = "Transfer-Encoding" + // Error strings a client may match on, so they have to read identically // whichever connector answers. Both take amqp091's wording: noMessageError is // what it returns for an ack, nack, or dead-letter naming a message it does @@ -1686,6 +1698,7 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai defer util.RecoverPanic() headers := natsToPbHeader(m.Headers()) + body := m.Data() // Synthesize the headers a RabbitMQ client receives that have no NATS // equivalent on the wire. Both come from the same metadata read. if md, err := m.Metadata(); err == nil { @@ -1706,6 +1719,20 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai } } + // STREAM-source parity with amqp091's streamSubscribe: undo a publisher's + // (or amqp091's own 1MiB-workaround) gzip compression rather than handing + // a consumer raw compressed bytes it never asked for. Queue/topic sources + // never decompress on either connector. + if source.GetType() == pb.Source_STREAM && headers[transferEncodingHeaderName] == "gzip" { + if decompressed, derr := decompressBody(body); derr != nil { + util.Logger.Debugf("natsjs: failed to decompress message on source %s: %s", + source.GetName(), derr.Error()) + } else { + body = decompressed + delete(headers, transferEncodingHeaderName) + } + } + // Proxy-side replacement for RabbitMQ headers-exchange routing. A failed // ack here only means the message will be redelivered and re-filtered — // harmless, but it inflates NumDelivered (and so the synthesized @@ -1720,7 +1747,7 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai uuid := util.GenUUID() bd.activeMessages.Add(uuid, inflightMsg{msg: m, subKey: subKey}) - pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: m.Data()} + pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: body} select { case out <- pbmsg: diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index e3308b0..e693685 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -4,6 +4,8 @@ package natsjs import ( + "bytes" + "compress/gzip" "context" "fmt" "net" @@ -3662,3 +3664,73 @@ func TestQueueDeliveryOmitsCurrentOffset(t *testing.T) { _, ok := m.GetHeaders()[currentOffsetHeaderName] assert.False(t, ok, "x-current-offset is a STREAM-only header on amqp091 too; a queue source must not carry it") } + +// TestStreamGzipBodyIsDecompressed: amqp091's streamSubscribe decompresses a +// gzip-tagged body and strips the header (amqp091.go:1286-1292) — a proxy-side +// step needed because arke's own STREAM publish path transparently compresses +// bodies over the RabbitMQ-Streams client's ~1MiB framing limit, but applied +// to any body carrying the header regardless of who compressed it. Without +// this, a natsjs STREAM consumer sees the raw compressed bytes: body +// corruption, not just a missing header. +func TestStreamGzipBodyIsDecompressed(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzstream", Subjects: []string{"e"}} + plain := []byte("the quick brown fox jumps over the lazy dog - repeated for compressibility - the quick brown fox") + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write(plain) + require.NoError(t, err) + require.NoError(t, gw.Close()) + + src := streamSource("events.gzstream.consumer", "events.gzstream", "first", "e") + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: buf.Bytes(), + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, plain, m.GetBody(), "a STREAM consumer must see the decompressed body, matching amqp091's streamSubscribe") + _, ok := m.GetHeaders()[transferEncodingHeaderName] + assert.False(t, ok, "the Transfer-Encoding header must be stripped once the body is decompressed, matching amqp091") +} + +// TestQueueGzipBodyPassesThroughUntouched: amqp091 only decompresses on the +// STREAM path; a QUEUE source's body and header must pass through exactly as +// published, on both connectors. +func TestQueueGzipBodyPassesThroughUntouched(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzqueue", Subjects: []string{"e"}} + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, gw.Close()) + gzipped := append([]byte(nil), buf.Bytes()...) + + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.gzqueue.consumer", "events.gzqueue", "e"), out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: gzipped, + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, gzipped, m.GetBody()) + assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName]) +} From 6f1144fff2ff8059aa1a62f57845bffe4dd9fc86 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 19:38:00 -0400 Subject: [PATCH 69/77] fix: inject trace-context headers on delivery, matching amqp091 amqp091's queueSubscribe starts (or continues, if the publisher already set traceparent/tracestate) a span for every delivered message and, when tracing is enabled, writes that span's W3C trace context back into the headers the consumer receives -- so a distributed trace always records arke's own "received from broker" hop, even from a publisher that set no trace headers at all. natsjs had zero tracing instrumentation, so that hop was invisible on this connector. Deliberately narrow in scope: amqp091 has roughly a dozen other tracing calls (spans around publish/ack/nack/subscribe-setup), which are arke's own observability instrumentation, not something a downstream client observes or a broker-parity question. Porting all of them would be a much larger, more invasive OTEL-instrumentation change for gaps nobody has flagged; this ports only the one consumer-visible span amqp091 actually adds on delivery. TestDeliveryInjectsTraceHeadersWhenTracingEnabled brackets the process-global tracing.Enabled() flag carefully (flips it via InitTracerProvider, restores the disabled state via defer) since other tests in this package assert exact header sets and would break under a leaked global; proven to fail (empty traceparent) with tracing.Enabled() short-circuited to false at the injection site. Signed-off-by: Michael Otteni --- doc/design/natsjs-connector.md | 1 + internal/provider/connectors/natsjs/natsjs.go | 21 ++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 40 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/doc/design/natsjs-connector.md b/doc/design/natsjs-connector.md index 04d7315..1172614 100644 --- a/doc/design/natsjs-connector.md +++ b/doc/design/natsjs-connector.md @@ -649,6 +649,7 @@ Legend: **Native** = NATS does it; **Proxy** = rebuilt in the connector; | Max message size | Partial | NATS caps a payload at the server's `max_payload` (1MB default) where RabbitMQ's `max_message_size` default is 16MB (see limitations). | | Publish / deliver rates in stats | Proxy | Sampled: counter deltas between `SourceStats` calls (see limitations). | | RabbitMQ management HTTP API | Drop | Replaced by the JetStream API over NATS itself. | +| Distributed tracing (`traceparent`/`tracestate`) | Native | Delivery starts (or continues) a span and writes its W3C trace context back into the consumed message, same as amqp091's `queueSubscribe`. | | Stream reader position header (`x-current-offset`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`; the message's own stream sequence in the `Offset` vocabulary. | | Gzip body passthrough (`Transfer-Encoding: gzip`) | Native | STREAM-source deliveries only, matching amqp091's `streamSubscribe`: a gzip-tagged body is decompressed and the header stripped. natsjs never compresses on publish — `max_payload` has no equivalent to the RabbitMQ-Streams client's ~1MiB ceiling amqp091 works around — so this only ever undoes a publisher's own compression. | diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 1abb38d..67967b3 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -32,6 +32,9 @@ import ( pb "github.com/sassoftware/arke/api" "github.com/sassoftware/arke/internal/provider" "github.com/sassoftware/arke/internal/util" + "github.com/sassoftware/arke/internal/util/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) const providerName = "natsjs" @@ -1745,6 +1748,24 @@ func (p *natsjsProvider) handleDelivery(ctx context.Context, bd *natsBrokerDetai return } + // Mirrors amqp091's queueSubscribe: start (or continue, if the publisher + // already set traceparent/tracestate) a span for this delivery and, when + // tracing is enabled, write its context back into the headers the + // consumer receives. Without this a distributed trace has no record of + // arke's own "received from broker" hop on natsjs, even though amqp091 + // always injects one. + _, span := tracing.SpanFromHeaders(ctx, headers, source.GetAddress().GetName()+" received from broker", trace.SpanKindInternal) + if tracing.Enabled() { + span.SetAttributes(attribute.String("source.name", source.GetName()), + attribute.String("messaging.client_id", bd.clientIdentifier)) + headers[tracing.HeaderTraceState] = span.SpanContext().TraceState().String() + headers[tracing.HeaderTraceParent] = fmt.Sprintf("00-%s-%s-%s", + span.SpanContext().TraceID().String(), + span.SpanContext().SpanID().String(), + span.SpanContext().TraceFlags()) + } + span.End() + uuid := util.GenUUID() bd.activeMessages.Add(uuid, inflightMsg{msg: m, subKey: subKey}) pbmsg := &pb.Message{Uuid: uuid, Headers: headers, Body: body} diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index e693685..d45627e 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "net" + "os" "strconv" "sync" "sync/atomic" @@ -19,6 +20,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" pb "github.com/sassoftware/arke/api" + "github.com/sassoftware/arke/internal/util/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -3734,3 +3736,41 @@ func TestQueueGzipBodyPassesThroughUntouched(t *testing.T) { assert.Equal(t, gzipped, m.GetBody()) assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName]) } + +// TestDeliveryInjectsTraceHeadersWhenTracingEnabled: mirrors amqp091's +// queueSubscribe, which starts (or continues) a span for every delivery and, +// when tracing is enabled, writes its W3C trace context back into the +// consumed message's headers — so a distributed trace shows arke's own +// "received from broker" hop even when the publisher set no trace headers at +// all. tracing.Enabled() is a process-global flag flipped by +// InitTracerProvider, so this test brackets it carefully and restores the +// disabled state before returning, since other tests in this package assert +// exact header sets and would break under a leaked global. +func TestDeliveryInjectsTraceHeadersWhenTracingEnabled(t *testing.T) { + require.NoError(t, os.Setenv(tracing.EnvTelemetryExporter, "stdout")) + require.NoError(t, os.Setenv(tracing.EnvOtelSdkDisabled, "false")) + _, err := tracing.InitTracerProvider() + require.NoError(t, err) + defer func() { + _ = os.Setenv(tracing.EnvOtelSdkDisabled, "true") + _, _ = tracing.InitTracerProvider() + }() + require.True(t, tracing.Enabled()) + + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.traced", Subjects: []string{"e"}} + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, queueSource("events.traced.consumer", "events.traced", "e"), out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("m")})) + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + + assert.NotEmpty(t, m.GetHeaders()[tracing.HeaderTraceParent], + "a consumed message must carry a traceparent when tracing is enabled, matching amqp091's queueSubscribe") + assert.Contains(t, m.GetHeaders()[tracing.HeaderTraceParent], "-", "traceparent must be the W3C dashed format") +} From 8e704723a1cd30a6a5f4fb48373ab98a09ff898e Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 20:06:01 -0400 Subject: [PATCH 70/77] fix: keep a failed Retry's message resolvable for the fallback nack Found by diffing this branch against upstream main: a sibling commit there (retry procedure hardening) added a fallback -- when the delayed- retry path (Retry) returns an error, the server now nacks the same uuid instead of leaving it (server.go). That fallback only works if the uuid is still resolvable when it fires. natsjs's Retry used takeMessage, which deletes the activeMessages entry before attempting NakWithDelay -- the same shape of bug DeadLetter had before its own fix (a46a8e2): if the nak fails, the uuid is already gone, so the new fallback nack finds nothing and errors again, and the message sits ack-pending until AckWait expires on its own rather than redelivering promptly. Retry now peeks the entry, marks it redeliverable (so the fallback nack means "put it back", not "reject" -- a freshly delivered message's redeliverOnNack defaults to false, and a failed retry is a request to try again), and only deletes it once NakWithDelay actually succeeds. Ports the matching 3-line addition to server.go's Ack/Nack dispatch, provider-agnostic and additive. TestRetryFailureKeepsMessageResolvable forces the failure the same way TestDeadLetterFailedTermKeepsMessageResolvable does (ack the underlying message directly, so NakWithDelay returns ErrMsgAlreadyAckd), and is proven to fail against the old takeMessage-based Retry before this fix. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 24 +++++++++-- .../provider/connectors/natsjs/natsjs_test.go | 41 +++++++++++++++++++ internal/server/server.go | 3 ++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 67967b3..2827228 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -1880,14 +1880,32 @@ func (p *natsjsProvider) Nack(ctx context.Context, uuid string) *pb.Error { // JetStream primitive that replaces RabbitMQ's per-message-TTL + dead-letter // retry-queue idiom; JetStream increments the delivery count for us, which // handleDelivery surfaces as x-retry-count. +// +// Mirrors DeadLetter's ordering: the server falls back to nacking this same +// uuid when Retry returns an error (server.go), and that fallback can only +// resolve anything while the uuid is still in activeMessages. Deleting it +// up front (as a plain takeMessage would) strands the message the moment +// NakWithDelay fails — neither nak'd nor resolvable — until AckWait expires +// on its own. markForRedelivery also makes that fallback nack mean "put it +// back", not "reject": a freshly delivered message's redeliverOnNack default +// is false, and a failed Retry is a request to try again, not to give up. func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, delay int32) *pb.Error { - m, perr := p.takeMessage(ctx, uuid) - if perr != nil { - return perr + bd, err := p.getBrokerDetails(ctx) + if err != nil { + return &pb.Error{Message: err.Error()} + } + mu, ok := bd.activeMessages.Get(uuid) + if !ok { + return &pb.Error{Message: fmt.Sprintf(noMessageError, uuid)} } + m := mu.(inflightMsg).msg + markForRedelivery(bd, uuid) if err := m.NakWithDelay(time.Duration(delay) * time.Second); err != nil { return &pb.Error{Message: err.Error()} } + bd.activeMu.Lock() + bd.activeMessages.Delete(uuid) + bd.activeMu.Unlock() return nil } diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index d45627e..b5e58c6 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -274,6 +274,47 @@ func TestRetrySynthesizesRetryCount(t *testing.T) { require.Nil(t, p.Ack(ctx, second.GetUuid())) } +// TestRetryFailureKeepsMessageResolvable: if NakWithDelay fails, Retry must +// leave the active-message entry in place instead of deleting it up front +// (which a plain takeMessage would). The server falls back to nacking this +// same uuid when Retry returns an error (server.go, mirroring the existing +// DeadLetter fallback) — a fallback that can only resolve anything while the +// uuid stays in activeMessages. Deleting it first strands the message with +// neither a nak nor a term having reached the server, until AckWait expires +// on its own instead of redelivering promptly. +func TestRetryFailureKeepsMessageResolvable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.retryfail", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("work")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.retryfail.consumer", "events.retryfail", "job") + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + + // Force NakWithDelay to fail: ack the underlying message directly, + // bypassing the connector's own bookkeeping, so the entry stays in + // activeMessages but the message itself is already resolved server-side. + bd := p.getBrokerDetailsByIdentifier("test-client") + mu, ok := bd.activeMessages.Get(m.GetUuid()) + require.True(t, ok) + require.NoError(t, mu.(inflightMsg).msg.Ack()) + + perr := p.Retry(ctx, src, m.GetUuid(), 1) + require.NotNil(t, perr, "Retry must surface the NakWithDelay failure") + + entry, stillThere := bd.activeMessages.Get(m.GetUuid()) + require.True(t, stillThere, + "a failed NakWithDelay must not remove the active-message entry the fallback nack needs") + assert.True(t, entry.(inflightMsg).redeliverOnNack, + "the fallback nack must put the message back, not term it, after a failed retry") +} + // TestNackDoesNotRedeliver pins Nack to amqp091's Delivery.Nack(requeue=false): // a nacked message is rejected, not put back. Naking it instead (JetStream's // Nak, which means redeliver now) turns one nacked message into an unbounded diff --git a/internal/server/server.go b/internal/server/server.go index 9c8ccf1..4d545be 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -344,6 +344,9 @@ consumeLoop: ackerr = &pb.Error{Message: "Uuid not set when acking/nacking"} } else if ackmsg.GetNack() && ackmsg.GetRequeueDelay() > 0 { // delayed retry ackerr = prov.Retry(ctx, source, ackmsg.GetUuid(), ackmsg.GetRequeueDelay()) + if ackerr != nil { + ackerr = prov.Nack(ctx, ackmsg.GetUuid()) + } } else if ackmsg.GetNack() { // Nack // dead letter if enabled, else Nack opts := source.GetOptions() From 5a247f859fcce3333d4ae549985e4b1bb0b00de2 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Tue, 21 Jul 2026 20:38:25 -0400 Subject: [PATCH 71/77] test(natsjs): cover header-filter/wire-escape composition and helper edge branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add regression tests for paths the existing suite left uncovered, each at the intersection of features added in separate sweeps or on a branch no prior test drove: - TestHeaderFilterMatchesWireEscapedHeader: a headers (FILTER) source whose match key is not an RFC 7230 token and whose value would be trimmed by http.Header.Write must still match exactly the messages a RabbitMQ headers exchange would. The filter runs on the decoded header map, so the wire escaping has to round-trip transparently before the match; proven to fail (delivery times out) with escaping disabled. The escaping tests and the header-filter tests never covered the composition together. - TestStreamGzipDecompressFailureKeepsRawBody: a STREAM body mislabelled Transfer-Encoding: gzip that is not valid gzip is forwarded raw with the header kept, matching amqp091's streamSubscribe. Only the success path was covered before. - TestDecompressBody: drives decompressBody's three outcomes directly — valid round-trip, non-gzip (reader-header error), and a truncated payload (the io.ReadAll error branch neither delivery test reached). - TestEvaluateFilters: add an x-match=any-all-miss case and a no-matches filter (the len(matches)==0 short-circuit), taking filterMatches to 100%. - TestFilterSubjectsForStreamSourceNoSubjects: a subjectless STREAM source reads the whole log (boundNothingSubjects' STREAM branch). - TestNatsToPbHeaderSkipsEmptyValueSlice: a raw NATS header key with no values is skipped, not indexed out of range. filterMatches / boundNothingSubjects / natsToPbHeader now 100%; decompressBody 88.9% (residual is the r.Close() error line, not deterministically triggerable); package 94.3%. No production code change. Signed-off-by: Michael Otteni --- .../connectors/natsjs/helpers_test.go | 74 ++++++++++++++++ .../provider/connectors/natsjs/natsjs_test.go | 85 +++++++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 439f2b6..711f053 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -5,6 +5,7 @@ package natsjs import ( "bytes" + "compress/gzip" "net/http" "strings" "testing" @@ -457,10 +458,57 @@ func TestEvaluateFilters(t *testing.T) { } assert.True(t, evaluateFilters([]*pb.Filter{any}, headers)) + // ANY where NOTHING hits -> fail. A present header with the wrong value does + // not count, exactly as a RabbitMQ headers exchange with x-match=any requires. + anyMiss := &pb.Filter{ + Type: pb.Filter_ANY, + Matches: []*pb.Match{ + {Name: "event-source", Value: "nope"}, + {Name: "region", Value: "eu"}, // present but wrong value + {Name: "absent", Value: "there"}, // not present at all + }, + } + assert.False(t, evaluateFilters([]*pb.Filter{anyMiss}, headers)) + + // A filter with NO matches passes everything: amqp091 still appends its + // (empty) binding table, and a headers binding with no arguments matches + // every message. Also the only thing that exercises the len(matches)==0 + // short-circuit in filterMatches. + assert.True(t, evaluateFilters([]*pb.Filter{{Type: pb.Filter_ALL}}, headers)) + assert.True(t, evaluateFilters([]*pb.Filter{{Type: pb.Filter_ANY}}, headers)) + // multiple filters are OR'd: second filter matches assert.True(t, evaluateFilters([]*pb.Filter{allMiss, all}, headers)) } +// TestDecompressBody exercises decompressBody's three outcomes directly: a +// valid gzip body round-trips; a body that is not gzip at all fails at the +// reader header; and a body with a valid gzip header but a truncated payload +// fails during the read, not at construction. The delivery-path test +// (TestStreamGzipBodyIsDecompressed) only ever drove the success case, and the +// stream mislabel test only the bad-header case. +func TestDecompressBody(t *testing.T) { + plain := []byte("payload worth compressing, repeated payload worth compressing") + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write(plain) + require.NoError(t, err) + require.NoError(t, gw.Close()) + gzipped := buf.Bytes() + + got, err := decompressBody(gzipped) + require.NoError(t, err) + assert.Equal(t, plain, got) + + _, err = decompressBody([]byte("not gzip at all")) + assert.Error(t, err, "a non-gzip body must error at the reader header") + + // Valid 10-byte header, payload cut short: NewReader succeeds, the inflate + // fails partway — the io.ReadAll error branch the success case never reaches. + _, err = decompressBody(gzipped[:len(gzipped)-6]) + assert.Error(t, err, "a truncated gzip stream must error during the read") +} + // TestFilterSubjectsForHeadersAddress pins that a headers address selects its // whole subject space regardless of the subjects it declares: RabbitMQ's // headers exchange routes on header arguments alone and never reads the @@ -489,6 +537,17 @@ func TestFilterSubjectsForHeadersAddress(t *testing.T) { assert.Equal(t, []string{"events.hdr.~.bound.one"}, filterSubjectsFor(topic)) } +// TestFilterSubjectsForStreamSourceNoSubjects: a STREAM source with no subjects +// reads the whole log — amqp091's streamSubscribe goes straight to the stream +// by name and declares no binding, so its reader sees everything. Selecting the +// whole address's captured subjects is the equivalent, and is the ordinary way +// to read a stream (a QUEUE/topic source with no subjects instead gets an +// unmatchable filter, per TestFilterSubjectsFor). +func TestFilterSubjectsForStreamSourceNoSubjects(t *testing.T) { + src := &pb.Source{Type: pb.Source_STREAM, Address: &pb.Address{Name: "events.orders"}} + assert.Equal(t, []string{"events.orders.~", "events.orders.~.>"}, filterSubjectsFor(src)) +} + // TestParentBindingSubjectsHeadersParent: the binding keys of an // address-to-address binding are matched by the PARENT exchange's type, so a // headers parent ignores them too — amqp091's ExchangeBind passes the child's @@ -538,6 +597,21 @@ func TestHeaderRoundTripPreservesEverything(t *testing.T) { assert.Equal(t, in, got, "every header must survive the round trip byte-for-byte") } +// TestNatsToPbHeaderSkipsEmptyValueSlice: a raw NATS header can carry a key +// with no values (nats.go's Header is map[string][]string, and a foreign +// publisher or library may leave a key present with an empty slice). +// natsToPbHeader must skip it rather than index vals[0] out of range, while +// keeping the well-formed entries alongside it. pbToNatsHeader never produces +// this shape (it always Sets a single value), so the round-trip tests never +// reach the guard. +func TestNatsToPbHeaderSkipsEmptyValueSlice(t *testing.T) { + in := nats.Header{ + "Present": []string{"here"}, + "Empty": []string{}, + } + assert.Equal(t, map[string]string{"Present": "here"}, natsToPbHeader(in)) +} + // TestHeaderEscapingOnlyWhereNeeded: conventional names and values reach the // broker verbatim, so ordinary traffic stays readable to a native NATS // consumer and only the entries NATS cannot carry are encoded. diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index b5e58c6..5fa0713 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -3307,6 +3307,56 @@ func TestHeadersAddressStillAppliesHeaderFilters(t *testing.T) { } } +// TestHeaderFilterMatchesWireEscapedHeader exercises the intersection of two +// features added at different times: the wire-format header escaping +// (pbToNatsHeader/natsToPbHeader — a name or value the NATS wire would drop or +// trim is base64-wrapped under X-Arke-Esc-) and proxy-side headers-exchange +// matching (evaluateFilters). A FILTER source that matches on a header whose +// NAME is not an RFC 7230 token (spaces, a colon) and whose VALUE would not +// survive http.Header.Write (leading/trailing spaces) must still match exactly +// the messages RabbitMQ's headers exchange would — the filter is evaluated on +// the DECODED header map, so the escape has to round-trip transparently before +// the match runs. If it did not, a headers source keyed on any non-token header +// would silently drop every message. Neither the escaping tests +// (TestHeadersSurviveARealBrokerRoundTrip) nor the filter tests +// (TestHeaderFilterDropsNonMatching) cover this composition on their own. +func TestHeaderFilterMatchesWireEscapedHeader(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // Name has a space and a colon (neither is a tchar); value has leading and + // trailing spaces (trimmed by the wire) — so both halves force escaping. + const hdrName, hdrVal = "x tenant:id", " acme " + addr := &pb.Address{Name: "events.escfilter", Type: pb.Address_FILTER, Subjects: []string{"k"}} + out := make(chan *pb.Message, 8) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, &pb.Source{ + Name: "events.escfilter.consumer", Type: pb.Source_QUEUE, Address: addr, + Filters: []*pb.Filter{{Type: pb.Filter_ALL, Matches: []*pb.Match{{Name: hdrName, Value: hdrVal}}}}, + Options: map[string]string{"Offset": "first"}, + }, out) + time.Sleep(500 * time.Millisecond) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: map[string]string{hdrName: hdrVal}, Body: []byte("match")})) + // Same escaped name, a value that differs only in the spaces the wire would + // have trimmed — proving the value round-trips byte-for-byte into the match. + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, Headers: map[string]string{hdrName: "acme"}, Body: []byte("nomatch")})) + + m := recv(t, out) + assert.Equal(t, "match", string(m.GetBody())) + assert.Equal(t, hdrVal, m.GetHeaders()[hdrName], + "the escaped header must decode byte-for-byte for the consumer, not just for the filter") + require.Nil(t, p.Ack(ctx, m.GetUuid())) + select { + case extra := <-out: + t.Fatalf("a value differing only by wire-trimmed spaces must not match; got %q", string(extra.GetBody())) + case <-time.After(2 * time.Second): + } +} + // TestBareStreamAssertionIssuesNoUpdate pins that asserting a stream that // already satisfies the requested config writes nothing. // @@ -3778,6 +3828,41 @@ func TestQueueGzipBodyPassesThroughUntouched(t *testing.T) { assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName]) } +// TestStreamGzipDecompressFailureKeepsRawBody: a STREAM body tagged +// Transfer-Encoding: gzip that is not in fact valid gzip must be delivered +// untouched — raw body and the header both preserved — rather than dropped or +// blanked. This matches amqp091's streamSubscribe, which on a decompress error +// logs at debug and forwards the original data with the header intact +// (amqp091.go decompress branch). natsjs never compresses on publish, so this +// failure path only ever fires for a foreign publisher's mislabelled body; the +// success path (TestStreamGzipBodyIsDecompressed) left it uncovered. +func TestStreamGzipDecompressFailureKeepsRawBody(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.gzbad", Subjects: []string{"e"}} + notGzip := []byte("this is plainly not gzip-framed data") + + src := streamSource("events.gzbad.consumer", "events.gzbad", "first", "e") + out := make(chan *pb.Message, 1) + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: addr, + Headers: map[string]string{transferEncodingHeaderName: "gzip"}, + Body: notGzip, + })) + + m := recv(t, out) + require.Nil(t, p.Ack(ctx, m.GetUuid())) + assert.Equal(t, notGzip, m.GetBody(), + "a body that fails to decompress must be forwarded raw, matching amqp091") + assert.Equal(t, "gzip", m.GetHeaders()[transferEncodingHeaderName], + "the Transfer-Encoding header must be kept when decompression fails, matching amqp091") +} + // TestDeliveryInjectsTraceHeadersWhenTracingEnabled: mirrors amqp091's // queueSubscribe, which starts (or continues) a span for every delivery and, // when tracing is enabled, writes its W3C trace context back into the From 1586951a7f8d81bdc70b095068b9497706007775 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 22 Jul 2026 11:05:02 -0400 Subject: [PATCH 72/77] test(natsjs): pin a short AckWait so the offset-pinning test cannot flake TestDurableOffsetPinnedAtCreation could fail intermittently with "timed out waiting for a message" on its re-subscribe. It needed two things to coincide: 1. The first subscription reads ahead and pulls m1 into flight before the test cancels it. PrefetchCount 1 makes this uncommon but not impossible -- once m0's handler returns, the puller is free to fetch m1. 2. The teardown nak that would make m1 promptly redeliverable does not take effect. releaseInFlight naks a durable's abandoned deliveries for exactly this reason, but it is best-effort: as its own comment notes, a nak that loses its race with the unsubscribe is redelivered into the dying pull request and sits unclaimed until AckWait. AckWait then defaults to 30s, which outlasts recv's 10s timeout, so the re-subscribe gave up before m1 came back. Reproduced deterministically by holding the first subscription open long enough to fetch m1 while suppressing the teardown nak; with those two conditions the assertion never completed, and pinning AckWait below recv's timeout makes it pass in ~0.5s. The subject of this test is the pinned start position, not redelivery latency, so it should not depend on that timer at all. TestTeardownReleasesInFlightMessages deliberately keeps the real 30s AckWait and uses the 10s-vs-30s gap as its assertion that the nak works, so the nak path stays covered and a genuine regression there is still caught. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 5fa0713..bd2368f 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -1934,6 +1934,16 @@ func TestSubscribeInvalidOffsetRejected(t *testing.T) { // not fail; it attaches to the existing durable and resumes from its stored ack // position, per the documented contract. func TestDurableOffsetPinnedAtCreation(t *testing.T) { + // The first subscription reads ahead, so it can pull m1 into flight before + // this test cancels it. releaseInFlight naks such messages precisely so + // they redeliver promptly, but that nak is best-effort: if it loses its + // race with the unsubscribe, the server holds m1 until AckWait instead. + // The production default (30s) outlasts recv's own 10s timeout, so when + // both of those happen at once the test failed spuriously. Pin an AckWait + // shorter than recv's timeout — the subject here is the pinned start + // position, not redelivery latency. + t.Setenv("NATSJS_ACK_WAIT", "2s") + s := runJetStreamServer(t) p, ctx := connectClient(t, s) From 02917f8e74bdd96e9afe8d4725d4869651daea43 Mon Sep 17 00:00:00 2001 From: Richard Sugg Date: Wed, 22 Jul 2026 11:30:46 -0400 Subject: [PATCH 73/77] test: fix source stats integration test Signed-off-by: Richard Sugg --- README.md | 380 ++++---------------------- api/arke.pb.go | 2 +- api/protobuf-spec/arke.proto | 2 +- doc/arke_protocol.md | 2 +- tests/integration/integration_test.go | 12 +- 5 files changed, 66 insertions(+), 332 deletions(-) diff --git a/README.md b/README.md index 61d9856..4487552 100644 --- a/README.md +++ b/README.md @@ -1,348 +1,76 @@ -# ![Arke message broker proxy](doc/images/arke_logo.png) +

+ NATS Logo +

-## Overview +[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)). NATS has over [40 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems. -Arke is a message broker proxy with a gRPC front-end interface that supports -pluggable back-end message brokers. Currently supported: **AMQP 0.9.1** -(e.g. RabbitMQ, including RabbitMQ Streams). +[![License][License-Image]][License-Url] [![Build][Build-Status-Image]][Build-Status-Url] [![Release][Release-Image]][Release-Url] [![Slack][Slack-Image]][Slack-Url] [![Coverage][Coverage-Image]][Coverage-Url] [![Docker Downloads][Docker-Image]][Docker-Url] [![GitHub Downloads][GitHub-Image]][Somsubhra-URL] [![CII Best Practices][CIIBestPractices-Image]][CIIBestPractices-Url] [![Artifact Hub][ArtifactHub-Image]][ArtifactHub-Url] -> In Greek mythology, Arke is the messenger for the Titans and is sometimes -> affiliated with the faded second rainbow seen in the shadow of the first. -> [Wikipedia: Arke](https://en.wikipedia.org/wiki/Arke) +## Documentation -Arke lets your application talk to a single gRPC interface while Arke handles -the broker-specific protocol on the back end. Swapping or upgrading a message -broker does not require changes to your application. This abstraction also gives -operators centralized control over which broker features are exposed and -how they are used, without requiring modifications to individual applications. +- [Official Website](https://nats.io) +- [Official Documentation](https://docs.nats.io) +- [FAQ](https://docs.nats.io/reference/faq) +- Watch [a video overview](https://rethink.synadia.com/episodes/1/) of NATS. +- Watch [this video from SCALE 13x](https://www.youtube.com/watch?v=sm63oAVPqAM) to learn more about its origin story and design philosophy. -```text -Your Application - │ gRPC - ▼ - [ Arke ] - │ AMQP 0.9.1 / Streams - ▼ - Message Broker - (e.g., RabbitMQ) -``` +## Contact -## Table of Contents +- [Twitter](https://twitter.com/nats_io): Follow us on Twitter! +- [Google Groups](https://groups.google.com/forum/#!forum/natsio): Where you can ask questions +- [Slack](https://natsio.slack.com): Click [here](https://slack.nats.io) to join. You can ask questions to our maintainers and to the rich and active community. -- [Overview](#overview) -- [Table of Contents](#table-of-contents) -- [Features](#features) -- [Requirements](#requirements) -- [Installation](#installation) - - [Build from Source](#build-from-source) -- [Configuration](#configuration) - - [Core](#core) -- [Running](#running) -- [gRPC API](#grpc-api) -- [TLS](#tls) - - [Server (Arke listener)](#server-arke-listener) - - [Back-end (Broker connection)](#back-end-broker-connection) -- [Rate Limiting](#rate-limiting) -- [Observability](#observability) -- [Development](#development) - - [Running Unit Tests](#running-unit-tests) - - [Running Integration Tests](#running-integration-tests) - - [Regenerating Protocol Buffers](#regenerating-protocol-buffers) - - [Linting](#linting) - - [Makefile Targets](#makefile-targets) -- [Design Documentation](#design-documentation) -- [Contributing](#contributing) -- [Security Policy](#security-policy) -- [License](#license) -- [Third-party dependencies](#third-party-dependencies) - ---- - -## Features - -- **gRPC front-end** – `Producer`, `Consumer`, and `Healthz` services. -- **Pluggable back-end providers** – Register new broker types without - changing the API. -- **AMQP 0.9.1 support** – Full publish, consume, ack/nack, retry, and - dead-letter workflows. RabbitMQ Streams are supported as a target type. -- **TLS** – Optional TLS on the gRPC listener with configurable - certificate paths. -- **Rate limiting** – Per-client token-bucket rate limiter applied to - `Connect`, `Publish`, and `Consume` RPCs. -- **Prometheus metrics** – HTTP metrics endpoint served on the same port - as gRPC via connection multiplexing. -- **OpenTelemetry tracing** – Trace context propagated through all RPCs - and forwarded to an OTLP gRPC collector. -- **Health service** – Standard gRPC health protocol plus a bidirectional - `Healthz` stream that reports CPU/memory availability. -- **Message deduplication** – `publish_id` + `publisher_name` fields on - `Message` enable deduplication on Streams. -- **Kubernetes awareness** – Monitors HPA replica counts and broadcasts - `GOAWAY` health signals to clients when scaling up. -- **Source options** – Per-subscription options including `MessageTTL`, - `Expires`, `DeadLetterAddress`, `DeadLetterSubject`, `Offset`, and - `ConsumerGroup`. -- **Header filtering** – `Filter`/`Match` messages on source subscriptions - for server-side message filtering. - ---- - -## Requirements - -- **Go 1.25+** -- A running AMQP 0.9.1-compatible broker (e.g., - [RabbitMQ](https://www.rabbitmq.com/)) for back-end connectivity. -- `protoc` + `protoc-gen-go` + `protoc-gen-go-grpc` – only required if - regenerating protobuf files. -- `golangci-lint` – only required for linting. -- Docker + Docker Compose – only required for integration tests. - ---- - -## Installation - -### Build from Source - -```bash -# Clone the repository -git clone -cd arke - -# Build for your current platform -make build - -# Or build for a specific platform -make linux # → build/linux/arke -make darwin # → build/darwin/arke -make windows # → build/windows/arke - -# Build for all platforms -make all -``` - ---- - -## Configuration - -Arke is configured entirely through environment variables. - -### Core - - -| Variable | Default | Description | -| ---------------- | -------- | --------------------------------------------------------------------------------------------------- | -| `ARKE_PORT` | `50051` | TCP port Arke listens on for both gRPC and HTTP (Prometheus). | -| `ARKE_CERT_FILE` | _(none)_ | Path to the PEM-encoded TLS certificate file. Required together with `ARKE_CERT_KEY` to enable TLS. | -| `ARKE_CERT_KEY` | _(none)_ | Path to the PEM-encoded TLS private key file. | - -> For the complete environment variable reference — including rate limiting, TLS, and observability variables — see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#environment-variables-reference). - - ---- - -## Running - - -```bash -# Minimal – plain TCP, no TLS, no rate limiting -ARKE_PORT=50051 ./build/darwin/arke - -# With TLS -ARKE_CERT_FILE=/path/to/cert.pem ARKE_CERT_KEY=/path/to/key.pem ./build/darwin/arke - -# CPU and heap profiling -./build/darwin/arke --cpuprofile cpu.out --memprofile mem.out -``` - - -The process handles `SIGINT` (Ctrl-C) for a graceful shutdown. - ---- - -## gRPC API - -The protobuf specification lives in -[api/protobuf-spec/arke.proto](api/protobuf-spec/arke.proto). -Generated protocol documentation is in -[doc/arke_protocol.md](doc/arke_protocol.md). - -Arke exposes three gRPC services (Producer, Consumer, and Healthz). Please -see protocol documentation for details. - ---- - -## TLS - -### Server (Arke listener) - -Set `ARKE_CERT_FILE` and `ARKE_CERT_KEY` to enable TLS on the gRPC -listener. The listener uses `h2` and `http/1.1` ALPN protocols so that -both gRPC and the Prometheus HTTP endpoint remain accessible on the same -port. - -### Back-end (Broker connection) - -Pass `tls: true` in `ConnectionConfiguration` when calling `Connect`. -Set `ARKE_TRUSTED_CA_CERTIFICATES_PEM_FILE` to supply additional -trusted CA certificates globally at the process level. - -Use `--tls-skip-verify` to disable certificate verification entirely -(not recommended for production). - - -For Kubernetes TLS secret mount and certificate rotation guidance, see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#tls-certificates-secret-mount). - ---- - -## Rate Limiting - -Arke implements a per-client token-bucket rate limiter applied to the -`Connect`, `Publish`, and `Consume` RPCs. Each client consumes one token -per call; the bucket refills on a configurable interval. By default -(`ARKE_RATE_LIMIT_ENFORCED=false`) violations are logged but allowed — -set `ARKE_RATE_LIMIT_ENFORCED=true` to reject over-limit requests with -`RESOURCE_EXHAUSTED`. - - -See the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#rate-limiting) for the full variable reference and tuning guidance. - ---- - -## Observability - -- **Prometheus metrics** – Available over HTTP on the same port as gRPC - via `cmux`. Scrape endpoint: `http://:/metrics` -- **OpenTelemetry tracing** – Traces exported via OTLP gRPC to - `OTEL_EXPORTER_OTLP_ENDPOINT` (default `localhost:4317`). Disable with - `OTEL_SDK_DISABLED=true`. -- **Health check** – Standard `grpc.health.v1.Health` for Kubernetes - probes, plus a custom `Healthz` bidirectional stream that reports - CPU/memory availability and emits `GOAWAY` signals during HPA scale-up. - - -For alert thresholds, key metric names, tracing header formats, and Kubernetes probe configuration see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#metrics-tracing-and-health). - ---- - -## Development - -### Running Unit Tests - -```bash -make test -``` - -This runs `golangci-lint` followed by all unit tests with coverage output. -An HTML coverage report is written to `coverage.html`. - -### Running Integration Tests - -Integration tests require Docker and Docker Compose and spin up a -RabbitMQ container. - - -```bash -# Build Arke for Linux and start RabbitMQ + Arke via Docker Compose, then run tests -make integration - -# TLS variant -make integration_test_tls - -# All integration test variants -make integration_all -``` - - -### Regenerating Protocol Buffers - -```bash -make generate -``` - -Requires `protoc`, `protoc-gen-go`, and `protoc-gen-go-grpc`. Generated -files are written to [api/](api/). Documentation is regenerated to -[doc/arke_protocol.md](doc/arke_protocol.md) if `protoc-gen-doc` is also -installed. - -### Linting - -```bash -make lint -``` +## Contributing -### Makefile Targets +If you are interested in contributing to NATS, read about our... -Run `make help` for a complete list of available targets. +- [Contributing guide](./CONTRIBUTING.md) +- [Report issues or propose Pull Requests](https://github.com/nats-io) ---- +[License-Url]: https://www.apache.org/licenses/LICENSE-2.0 +[License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg +[Docker-Image]: https://img.shields.io/docker/pulls/_/nats.svg +[Docker-Url]: https://hub.docker.com/_/nats +[Slack-Image]: https://img.shields.io/badge/chat-on%20slack-green +[Slack-Url]: https://slack.nats.io +[Fossa-Url]: https://app.fossa.io/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server?ref=badge_shield +[Fossa-Image]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server.svg?type=shield +[Build-Status-Url]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml +[Build-Status-Image]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml/badge.svg?branch=main +[Release-Url]: https://github.com/nats-io/nats-server/releases/latest +[Release-Image]: https://img.shields.io/github/v/release/nats-io/nats-server +[Coverage-Url]: https://coveralls.io/r/nats-io/nats-server?branch=main +[Coverage-image]: https://coveralls.io/repos/github/nats-io/nats-server/badge.svg?branch=main +[ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-server +[ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-server +[CIIBestPractices-Url]: https://bestpractices.coreinfrastructure.org/projects/1895 +[CIIBestPractices-Image]: https://bestpractices.coreinfrastructure.org/projects/1895/badge +[ArtifactHub-Url]: https://artifacthub.io/packages/helm/nats/nats +[ArtifactHub-Image]: https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/nats +[GitHub-Release]: https://github.com/nats-io/nats-server/releases/ +[GitHub-Image]: https://img.shields.io/github/downloads/nats-io/nats-server/total.svg?logo=github +[Somsubhra-url]: https://somsubhra.github.io/github-release-stats/?username=nats-io&repository=nats-server -## Design Documentation +## Roadmap -Detailed design and operational documents live in -[`doc/design/`](doc/design/), including narrative architecture notes -and C4 diagram views. +The NATS product roadmap can be found [here](https://nats.io/about/#roadmap). - -| Document | Description | -| --- | --- | -| [Architecture Overview](doc/design/architecture-overview.md) | Component map, data-flow diagrams, key design decisions, and repository structure | -| [Architecture C4 Diagrams](doc/design/architecture-c4.md) | System-context, container, component, and code-level C4 views for the current server and connector architecture | -| [Client Publish / Consume Guide](doc/design/client-publish-consume.md) | How application clients connect to Arke, publish messages, consume deliveries, and ack or retry processing | -| [Connection and Message Lifecycle](doc/design/connection-message-lifecycle.md) | Detailed session phases, goroutine topology, ack/nack state machine, and GOAWAY flow | -| [Provider/Connector Interface Contract](doc/design/provider-connector-interface.md) | Interface specification and guide for adding new broker backends | -| [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md) | Full environment variable reference, Kubernetes deployment checklist, observability setup, and troubleshooting | -| [Protocol Reference](doc/arke_protocol.md) | Auto-generated reference for all protobuf messages, fields, enums, and gRPC service methods | - +## Adopters ---- +Who uses NATS? See our [list of users](https://nats.io/#who-uses-nats) on [https://nats.io](https://nats.io). -## Contributing +## Security -Maintainers are accepting patches and contributions to this project. -Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details about submitting -contributions to this project. +### Security Audit ---- +A third party security audit was performed by Trail of Bits following engagement by the Open Source Technology Improvement Fund (OSTIF). You can see the [full report from April 2025 here](https://github.com/trailofbits/publications/blob/master/reviews/2025-04-ostif-nats-securityreview.pdf). -## Security Policy +### Reporting Security Vulnerabilities -Please see our [Security Policy](SECURITY.md) for details. +If you've found a vulnerability or a potential vulnerability in the NATS server, please let us know at +[nats-security](mailto:security@nats.io). ## License -This project is licensed under the [Apache 2.0 License](LICENSE). - - -This project's logo features the Go Gopher mascot, [created by Renee French](https://go.dev/brand) and licensed under the [Creative Commons 4.0 Attribution License](https://creativecommons.org/licenses/by/4.0/). - - ---- - -## Third-party dependencies - - -| Dependency | License | -| --- | --- | -| `github.com/KimMachineGun/automemlimit` | [LICENSE](https://github.com/KimMachineGun/automemlimit/blob/main/LICENSE) | -| `github.com/hashicorp/go-metrics` | [LICENSE](https://github.com/hashicorp/go-metrics/blob/master/LICENSE) | -| `github.com/google/uuid` | [LICENSE](https://github.com/google/uuid/blob/master/LICENSE) | -| `github.com/grpc-ecosystem/go-grpc-middleware/v2` | [LICENSE](https://github.com/grpc-ecosystem/go-grpc-middleware/blob/main/LICENSE) | -| `github.com/prometheus/client_golang` | [LICENSE](https://github.com/prometheus/client_golang/blob/main/LICENSE) | -| `github.com/rabbitmq/amqp091-go` | [LICENSE](https://github.com/rabbitmq/amqp091-go/blob/main/LICENSE) | -| `github.com/rabbitmq/rabbitmq-stream-go-client` | [LICENSE](https://github.com/rabbitmq/rabbitmq-stream-go-client/blob/main/LICENSE) | -| `github.com/rs/zerolog` | [LICENSE](https://github.com/rs/zerolog/blob/master/LICENSE) | -| `github.com/soheilhy/cmux` | [LICENSE](https://github.com/soheilhy/cmux/blob/master/LICENSE) | -| `github.com/stretchr/testify` | [LICENSE](https://github.com/stretchr/testify/blob/master/LICENSE) | -| `go.opentelemetry.io/otel` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | -| `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | -| `go.opentelemetry.io/otel/exporters/stdout/stdouttrace` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | -| `go.opentelemetry.io/otel/sdk` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | -| `go.opentelemetry.io/otel/trace` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | -| `golang.org/x/text` | [LICENSE](https://github.com/golang/text/blob/master/LICENSE) | -| `golang.org/x/time` | [LICENSE](https://github.com/golang/time/blob/master/LICENSE) | -| `google.golang.org/grpc` | [LICENSE](https://github.com/grpc/grpc-go/blob/master/LICENSE) | -| `google.golang.org/protobuf` | [LICENSE](https://github.com/protocolbuffers/protobuf-go/blob/master/LICENSE) | -| `k8s.io/api` | [LICENSE](https://github.com/kubernetes/api/blob/master/LICENSE) | -| `k8s.io/apimachinery` | [LICENSE](https://github.com/kubernetes/apimachinery/blob/master/LICENSE) | -| `k8s.io/client-go` | [LICENSE](https://github.com/kubernetes/client-go/blob/master/LICENSE) | - +Unless otherwise noted, the NATS source files are distributed +under the Apache Version 2.0 license found in the LICENSE file. diff --git a/api/arke.pb.go b/api/arke.pb.go index 9195f64..df49f5d 100644 --- a/api/arke.pb.go +++ b/api/arke.pb.go @@ -232,7 +232,7 @@ type ConnectionConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // Broker hostname or IP address. Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` // Broker port. - Provider string `protobuf:"bytes,3,opt,name=provider,proto3" json:"provider,omitempty"` // Provider type, currently only amqp091. + Provider string `protobuf:"bytes,3,opt,name=provider,proto3" json:"provider,omitempty"` // Provider type, currently only amqp091, and natsjs. Tenant string `protobuf:"bytes,4,opt,name=tenant,proto3" json:"tenant,omitempty"` // Tenant name for this connection. Tenant is not required Credentials *Credentials `protobuf:"bytes,5,opt,name=credentials,proto3" json:"credentials,omitempty"` // Authentication credentials. Tls bool `protobuf:"varint,8,opt,name=tls,proto3" json:"tls,omitempty"` // Should this provider connection use TLS. diff --git a/api/protobuf-spec/arke.proto b/api/protobuf-spec/arke.proto index e107ed1..8e331fa 100644 --- a/api/protobuf-spec/arke.proto +++ b/api/protobuf-spec/arke.proto @@ -22,7 +22,7 @@ package arke; message ConnectionConfiguration { string host = 1; // Broker hostname or IP address. int32 port = 2; // Broker port. - string provider = 3; // Provider type, currently only amqp091. + string provider = 3; // Provider type, currently only amqp091, and natsjs. string tenant = 4; // Tenant name for this connection. Tenant is not required Credentials credentials = 5; // Authentication credentials. bool tls = 8; // Should this provider connection use TLS. diff --git a/doc/arke_protocol.md b/doc/arke_protocol.md index 5310118..3e0af8d 100644 --- a/doc/arke_protocol.md +++ b/doc/arke_protocol.md @@ -100,7 +100,7 @@ RabbitMQ and Kafka. | ----- | ---- | ----- | ----------- | | host | [string](#string) | | Broker hostname or IP address. | | port | [int32](#int32) | | Broker port. | -| provider | [string](#string) | | Provider type, currently only amqp091. | +| provider | [string](#string) | | Provider type, currently only amqp091, and natsjs. | | tenant | [string](#string) | | Tenant name for this connection. Tenant is not required | | credentials | [Credentials](#arke-Credentials) | | Authentication credentials. | | tls | [bool](#bool) | | Should this provider connection use TLS. | diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 34530c5..b8fe19a 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -3694,7 +3694,7 @@ func Test_SourceStatsNoStream(t *testing.T) { defer c.Disconnect(ctx, &pb.Empty{}) connResp, err := c.Connect(ctx, connConfig) - assert.Nil(t, err) + assert.Nil(t, err, "got an error: %+v", err) assert.NotNil(t, connResp) assert.True(t, connResp.GetSuccess()) @@ -3705,7 +3705,13 @@ func Test_SourceStatsNoStream(t *testing.T) { assert.Nil(t, ssErr) assert.NotNil(t, stats) assert.NotNil(t, stats.Error) - assert.Contains(t, stats.Error.GetMessage(), "request returned a 404") + brokerResponses := map[string]string{ + "amqp091": "request returned a 404", + "natsjs": "stream not found", + } + notFoundResponse := brokerResponses[connConfig.GetProvider()] + + assert.Contains(t, stats.Error.GetMessage(), notFoundResponse) // verified 404 when no created yet, now create it and get stats. // we should get an "Offset not found" error in the stats.Error @@ -3722,7 +3728,7 @@ func Test_SourceStatsNoStream(t *testing.T) { assert.Nil(t, ssErr) assert.NotNil(t, stats) // emsg := stats.Error.GetMessage() - if strings.Contains(stats.GetError().GetMessage(), "request returned a 404") { + if strings.Contains(stats.GetError().GetMessage(), notFoundResponse) { time.Sleep(1 * time.Second) continue } From 88d8988ecf9e870712908bec7b1d6b31a46ea79e Mon Sep 17 00:00:00 2001 From: Richard Sugg Date: Wed, 22 Jul 2026 13:32:48 -0400 Subject: [PATCH 74/77] fix: readme Signed-off-by: Richard Sugg --- README.md | 380 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 326 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 4487552..4f93e07 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,348 @@ -

- NATS Logo -

+# ![Arke message broker proxy](doc/images/arke_logo.png) -[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)). NATS has over [40 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems. +## Overview -[![License][License-Image]][License-Url] [![Build][Build-Status-Image]][Build-Status-Url] [![Release][Release-Image]][Release-Url] [![Slack][Slack-Image]][Slack-Url] [![Coverage][Coverage-Image]][Coverage-Url] [![Docker Downloads][Docker-Image]][Docker-Url] [![GitHub Downloads][GitHub-Image]][Somsubhra-URL] [![CII Best Practices][CIIBestPractices-Image]][CIIBestPractices-Url] [![Artifact Hub][ArtifactHub-Image]][ArtifactHub-Url] +Arke is a message broker proxy with a gRPC front-end interface that supports +pluggable back-end message brokers. Currently supported: **AMQP 0.9.1** +(e.g. RabbitMQ, including RabbitMQ Streams). -## Documentation +> In Greek mythology, Arke is the messenger for the Titans and is sometimes +> affiliated with the faded second rainbow seen in the shadow of the first. +> [Wikipedia: Arke](https://en.wikipedia.org/wiki/Arke) -- [Official Website](https://nats.io) -- [Official Documentation](https://docs.nats.io) -- [FAQ](https://docs.nats.io/reference/faq) -- Watch [a video overview](https://rethink.synadia.com/episodes/1/) of NATS. -- Watch [this video from SCALE 13x](https://www.youtube.com/watch?v=sm63oAVPqAM) to learn more about its origin story and design philosophy. +Arke lets your application talk to a single gRPC interface while Arke handles +the broker-specific protocol on the back end. Swapping or upgrading a message +broker does not require changes to your application. This abstraction also gives +operators centralized control over which broker features are exposed and +how they are used, without requiring modifications to individual applications. -## Contact +```text +Your Application + │ gRPC + ▼ + [ Arke ] + │ AMQP 0.9.1 / Streams + ▼ + Message Broker + (e.g., RabbitMQ) +``` -- [Twitter](https://twitter.com/nats_io): Follow us on Twitter! -- [Google Groups](https://groups.google.com/forum/#!forum/natsio): Where you can ask questions -- [Slack](https://natsio.slack.com): Click [here](https://slack.nats.io) to join. You can ask questions to our maintainers and to the rich and active community. +## Table of Contents -## Contributing +- [Overview](#overview) +- [Table of Contents](#table-of-contents) +- [Features](#features) +- [Requirements](#requirements) +- [Installation](#installation) + - [Build from Source](#build-from-source) +- [Configuration](#configuration) + - [Core](#core) +- [Running](#running) +- [gRPC API](#grpc-api) +- [TLS](#tls) + - [Server (Arke listener)](#server-arke-listener) + - [Back-end (Broker connection)](#back-end-broker-connection) +- [Rate Limiting](#rate-limiting) +- [Observability](#observability) +- [Development](#development) + - [Running Unit Tests](#running-unit-tests) + - [Running Integration Tests](#running-integration-tests) + - [Regenerating Protocol Buffers](#regenerating-protocol-buffers) + - [Linting](#linting) + - [Makefile Targets](#makefile-targets) +- [Design Documentation](#design-documentation) +- [Contributing](#contributing) +- [Security Policy](#security-policy) +- [License](#license) +- [Third-party dependencies](#third-party-dependencies) + +--- + +## Features + +- **gRPC front-end** – `Producer`, `Consumer`, and `Healthz` services. +- **Pluggable back-end providers** – Register new broker types without + changing the API. +- **AMQP 0.9.1 support** – Full publish, consume, ack/nack, retry, and + dead-letter workflows. RabbitMQ Streams are supported as a target type. +- **TLS** – Optional TLS on the gRPC listener with configurable + certificate paths. +- **Rate limiting** – Per-client token-bucket rate limiter applied to + `Connect`, `Publish`, and `Consume` RPCs. +- **Prometheus metrics** – HTTP metrics endpoint served on the same port + as gRPC via connection multiplexing. +- **OpenTelemetry tracing** – Trace context propagated through all RPCs + and forwarded to an OTLP gRPC collector. +- **Health service** – Standard gRPC health protocol plus a bidirectional + `Healthz` stream that reports CPU/memory availability. +- **Message deduplication** – `publish_id` + `publisher_name` fields on + `Message` enable deduplication on Streams. +- **Kubernetes awareness** – Monitors HPA replica counts and broadcasts + `GOAWAY` health signals to clients when scaling up. +- **Source options** – Per-subscription options including `MessageTTL`, + `Expires`, `DeadLetterAddress`, `DeadLetterSubject`, `Offset`, and + `ConsumerGroup`. +- **Header filtering** – `Filter`/`Match` messages on source subscriptions + for server-side message filtering. + +--- + +## Requirements + +- **Go 1.25+** +- A running AMQP 0.9.1-compatible broker (e.g., + [RabbitMQ](https://www.rabbitmq.com/)) for back-end connectivity. +- `protoc` + `protoc-gen-go` + `protoc-gen-go-grpc` – only required if + regenerating protobuf files. +- `golangci-lint` – only required for linting. +- Docker + Docker Compose – only required for integration tests. + +--- + +## Installation + +### Build from Source + +```bash +# Clone the repository +git clone +cd arke + +# Build for your current platform +make build + +# Or build for a specific platform +make linux # → build/linux/arke +make darwin # → build/darwin/arke +make windows # → build/windows/arke + +# Build for all platforms +make all +``` + +--- + +## Configuration + +Arke is configured entirely through environment variables. + +### Core + + +| Variable | Default | Description | +| ---------------- | -------- | --------------------------------------------------------------------------------------------------- | +| `ARKE_PORT` | `50051` | TCP port Arke listens on for both gRPC and HTTP (Prometheus). | +| `ARKE_CERT_FILE` | _(none)_ | Path to the PEM-encoded TLS certificate file. Required together with `ARKE_CERT_KEY` to enable TLS. | +| `ARKE_CERT_KEY` | _(none)_ | Path to the PEM-encoded TLS private key file. | + +> For the complete environment variable reference — including rate limiting, TLS, and observability variables — see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#environment-variables-reference). + + +--- + +## Running + + +```bash +# Minimal – plain TCP, no TLS, no rate limiting +ARKE_PORT=50051 ./build/darwin/arke + +# With TLS +ARKE_CERT_FILE=/path/to/cert.pem ARKE_CERT_KEY=/path/to/key.pem ./build/darwin/arke + +# CPU and heap profiling +./build/darwin/arke --cpuprofile cpu.out --memprofile mem.out +``` + + +The process handles `SIGINT` (Ctrl-C) for a graceful shutdown. + +--- + +## gRPC API + +The protobuf specification lives in +[api/protobuf-spec/arke.proto](api/protobuf-spec/arke.proto). +Generated protocol documentation is in +[doc/arke_protocol.md](doc/arke_protocol.md). + +Arke exposes three gRPC services (Producer, Consumer, and Healthz). Please +see protocol documentation for details. + +--- + +## TLS + +### Server (Arke listener) -If you are interested in contributing to NATS, read about our... +Set `ARKE_CERT_FILE` and `ARKE_CERT_KEY` to enable TLS on the gRPC +listener. The listener uses `h2` and `http/1.1` ALPN protocols so that +both gRPC and the Prometheus HTTP endpoint remain accessible on the same +port. -- [Contributing guide](./CONTRIBUTING.md) -- [Report issues or propose Pull Requests](https://github.com/nats-io) +### Back-end (Broker connection) -[License-Url]: https://www.apache.org/licenses/LICENSE-2.0 -[License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg -[Docker-Image]: https://img.shields.io/docker/pulls/_/nats.svg -[Docker-Url]: https://hub.docker.com/_/nats -[Slack-Image]: https://img.shields.io/badge/chat-on%20slack-green -[Slack-Url]: https://slack.nats.io -[Fossa-Url]: https://app.fossa.io/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server?ref=badge_shield -[Fossa-Image]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server.svg?type=shield -[Build-Status-Url]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml -[Build-Status-Image]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml/badge.svg?branch=main -[Release-Url]: https://github.com/nats-io/nats-server/releases/latest -[Release-Image]: https://img.shields.io/github/v/release/nats-io/nats-server -[Coverage-Url]: https://coveralls.io/r/nats-io/nats-server?branch=main -[Coverage-image]: https://coveralls.io/repos/github/nats-io/nats-server/badge.svg?branch=main -[ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-server -[ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-server -[CIIBestPractices-Url]: https://bestpractices.coreinfrastructure.org/projects/1895 -[CIIBestPractices-Image]: https://bestpractices.coreinfrastructure.org/projects/1895/badge -[ArtifactHub-Url]: https://artifacthub.io/packages/helm/nats/nats -[ArtifactHub-Image]: https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/nats -[GitHub-Release]: https://github.com/nats-io/nats-server/releases/ -[GitHub-Image]: https://img.shields.io/github/downloads/nats-io/nats-server/total.svg?logo=github -[Somsubhra-url]: https://somsubhra.github.io/github-release-stats/?username=nats-io&repository=nats-server +Pass `tls: true` in `ConnectionConfiguration` when calling `Connect`. +Set `ARKE_TRUSTED_CA_CERTIFICATES_PEM_FILE` to supply additional +trusted CA certificates globally at the process level. -## Roadmap +Use `--tls-skip-verify` to disable certificate verification entirely +(not recommended for production). -The NATS product roadmap can be found [here](https://nats.io/about/#roadmap). + +For Kubernetes TLS secret mount and certificate rotation guidance, see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#tls-certificates-secret-mount). -## Adopters +--- -Who uses NATS? See our [list of users](https://nats.io/#who-uses-nats) on [https://nats.io](https://nats.io). +## Rate Limiting -## Security +Arke implements a per-client token-bucket rate limiter applied to the +`Connect`, `Publish`, and `Consume` RPCs. Each client consumes one token +per call; the bucket refills on a configurable interval. By default +(`ARKE_RATE_LIMIT_ENFORCED=false`) violations are logged but allowed — +set `ARKE_RATE_LIMIT_ENFORCED=true` to reject over-limit requests with +`RESOURCE_EXHAUSTED`. -### Security Audit + +See the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#rate-limiting) for the full variable reference and tuning guidance. -A third party security audit was performed by Trail of Bits following engagement by the Open Source Technology Improvement Fund (OSTIF). You can see the [full report from April 2025 here](https://github.com/trailofbits/publications/blob/master/reviews/2025-04-ostif-nats-securityreview.pdf). +--- -### Reporting Security Vulnerabilities +## Observability -If you've found a vulnerability or a potential vulnerability in the NATS server, please let us know at -[nats-security](mailto:security@nats.io). +- **Prometheus metrics** – Available over HTTP on the same port as gRPC + via `cmux`. Scrape endpoint: `http://:/metrics` +- **OpenTelemetry tracing** – Traces exported via OTLP gRPC to + `OTEL_EXPORTER_OTLP_ENDPOINT` (default `localhost:4317`). Disable with + `OTEL_SDK_DISABLED=true`. +- **Health check** – Standard `grpc.health.v1.Health` for Kubernetes + probes, plus a custom `Healthz` bidirectional stream that reports + CPU/memory availability and emits `GOAWAY` signals during HPA scale-up. + + +For alert thresholds, key metric names, tracing header formats, and Kubernetes probe configuration see the [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md#metrics-tracing-and-health). + +--- + +## Development + +### Running Unit Tests + +```bash +make test +``` + +This runs `golangci-lint` followed by all unit tests with coverage output. +An HTML coverage report is written to `coverage.html`. + +### Running Integration Tests + +Integration tests require Docker and Docker Compose and spin up a +RabbitMQ container. + + +```bash +# Build Arke for Linux and start RabbitMQ + Arke via Docker Compose, then run tests +make integration + +# TLS variant +make integration_test_tls + +# All integration test variants +make integration_all +``` + + +### Regenerating Protocol Buffers + +```bash +make generate +``` + +Requires `protoc`, `protoc-gen-go`, and `protoc-gen-go-grpc`. Generated +files are written to [api/](api/). Documentation is regenerated to +[doc/arke_protocol.md](doc/arke_protocol.md) if `protoc-gen-doc` is also +installed. + +### Linting + +```bash +make lint +``` + +### Makefile Targets + +Run `make help` for a complete list of available targets. + +--- + +## Design Documentation + +Detailed design and operational documents live in +[`doc/design/`](doc/design/), including narrative architecture notes +and C4 diagram views. + + +| Document | Description | +| --- | --- | +| [Architecture Overview](doc/design/architecture-overview.md) | Component map, data-flow diagrams, key design decisions, and repository structure | +| [Architecture C4 Diagrams](doc/design/architecture-c4.md) | System-context, container, component, and code-level C4 views for the current server and connector architecture | +| [Client Publish / Consume Guide](doc/design/client-publish-consume.md) | How application clients connect to Arke, publish messages, consume deliveries, and ack or retry processing | +| [Connection and Message Lifecycle](doc/design/connection-message-lifecycle.md) | Detailed session phases, goroutine topology, ack/nack state machine, and GOAWAY flow | +| [Provider/Connector Interface Contract](doc/design/provider-connector-interface.md) | Interface specification and guide for adding new broker backends | +| [Deployment and Operations Runbook](doc/design/deployment-operations-runbook.md) | Full environment variable reference, Kubernetes deployment checklist, observability setup, and troubleshooting | +| [Protocol Reference](doc/arke_protocol.md) | Auto-generated reference for all protobuf messages, fields, enums, and gRPC service methods | + + +--- + +## Contributing + +Maintainers are accepting patches and contributions to this project. +Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details about submitting +contributions to this project. + +--- + +## Security Policy + +Please see our [Security Policy](SECURITY.md) for details. ## License -Unless otherwise noted, the NATS source files are distributed -under the Apache Version 2.0 license found in the LICENSE file. +This project is licensed under the [Apache 2.0 License](LICENSE). + + +This project's logo features the Go Gopher mascot, [created by Renee French](https://go.dev/brand) and licensed under the [Creative Commons 4.0 Attribution License](https://creativecommons.org/licenses/by/4.0/). + + +--- + +## Third-party dependencies + + +| Dependency | License | +| --- | --- | +| `github.com/KimMachineGun/automemlimit` | [LICENSE](https://github.com/KimMachineGun/automemlimit/blob/main/LICENSE) | +| `github.com/hashicorp/go-metrics` | [LICENSE](https://github.com/hashicorp/go-metrics/blob/master/LICENSE) | +| `github.com/google/uuid` | [LICENSE](https://github.com/google/uuid/blob/master/LICENSE) | +| `github.com/grpc-ecosystem/go-grpc-middleware/v2` | [LICENSE](https://github.com/grpc-ecosystem/go-grpc-middleware/blob/main/LICENSE) | +| `github.com/prometheus/client_golang` | [LICENSE](https://github.com/prometheus/client_golang/blob/main/LICENSE) | +| `github.com/rabbitmq/amqp091-go` | [LICENSE](https://github.com/rabbitmq/amqp091-go/blob/main/LICENSE) | +| `github.com/rabbitmq/rabbitmq-stream-go-client` | [LICENSE](https://github.com/rabbitmq/rabbitmq-stream-go-client/blob/main/LICENSE) | +| `github.com/rs/zerolog` | [LICENSE](https://github.com/rs/zerolog/blob/master/LICENSE) | +| `github.com/soheilhy/cmux` | [LICENSE](https://github.com/soheilhy/cmux/blob/master/LICENSE) | +| `github.com/stretchr/testify` | [LICENSE](https://github.com/stretchr/testify/blob/master/LICENSE) | +| `go.opentelemetry.io/otel` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | +| `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | +| `go.opentelemetry.io/otel/exporters/stdout/stdouttrace` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | +| `go.opentelemetry.io/otel/sdk` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | +| `go.opentelemetry.io/otel/trace` | [LICENSE](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE) | +| `golang.org/x/text` | [LICENSE](https://github.com/golang/text/blob/master/LICENSE) | +| `golang.org/x/time` | [LICENSE](https://github.com/golang/time/blob/master/LICENSE) | +| `google.golang.org/grpc` | [LICENSE](https://github.com/grpc/grpc-go/blob/master/LICENSE) | +| `google.golang.org/protobuf` | [LICENSE](https://github.com/protocolbuffers/protobuf-go/blob/master/LICENSE) | +| `k8s.io/api` | [LICENSE](https://github.com/kubernetes/api/blob/master/LICENSE) | +| `k8s.io/apimachinery` | [LICENSE](https://github.com/kubernetes/apimachinery/blob/master/LICENSE) | +| `k8s.io/client-go` | [LICENSE](https://github.com/kubernetes/client-go/blob/master/LICENSE) | + \ No newline at end of file From 995e7d08c7fcd5c1a9fbf2bc908a2099a6a15f3d Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 22 Jul 2026 17:01:19 -0400 Subject: [PATCH 75/77] fix: let a late-attaching dead-letter reader see earlier dead letters A consumer that attaches to a source's dead-letter queue after messages have been dead-lettered received none of them. Dead-lettering itself worked -- the copies were stored in the dead-letter stream -- but the reader could not see them. A dead-letter reader is transient by convention, which makes it an ephemeral consumer starting at DeliverNew: the stream tail, behind the very messages it exists to read. The amqp091 connector does not have this problem, and the reason is not a broker feature. Its setupDeadLetter eagerly declares a queue named .dlq bound to the dead-letter exchange as soon as a source declares a DeadLetterAddress, before anything can be dead-lettered, so that queue accumulates from declare time and a reader attaching later finds the messages waiting. Clients attach by that conventional name. So the faithful start position is neither the tail at attach time (what this connector used, which misses everything) nor DeliverAll (which would replay history a transient queue never held). It is the tail as of declare time. Mirror the other connector: pre-declare a durable consumer for the dead-letter queue when a source declares its DeadLetterAddress, and let a later transient reader of that name adopt it and inherit the position it has been holding. Best-effort, as it is there: a failure to pre-declare warns and leaves dead-lettering itself working. Two deliberate choices: Adoption is gated on marker metadata, not on the name alone, so a transient source can never hijack an unrelated durable that happens to share its name and take over its pending messages and ack position. The marker is carried forward when attaching, because attaching re-asserts the whole consumer config and one omitting Metadata erases it -- after which the queue is unadoptable and the next reader silently falls back to the tail. The pre-declared consumer expires after NATSJS_DLQ_DECLARE_TTL (default 1h) where the amqp091 queue never expires. Exact parity would leak: clients commonly name transient consumers with a fresh UUID, so a transient source with a DeadLetterAddress would strand one permanent durable per connection. Set NATSJS_DLQ_DECLARE_TTL=0 for exact parity where source names are stable. Signed-off-by: Michael Otteni --- .../provider/connectors/natsjs/helpers.go | 11 + .../connectors/natsjs/helpers_test.go | 19 ++ internal/provider/connectors/natsjs/natsjs.go | 194 +++++++++++- .../provider/connectors/natsjs/natsjs_test.go | 281 ++++++++++++++++++ 4 files changed, 504 insertions(+), 1 deletion(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 2144fd8..0a8d27f 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -591,6 +591,17 @@ func subjectSubsumes(wide, narrow string) bool { // auto-delete/exclusive/TEMPORARY sources are treated as the transient ones. A // non-transient QUEUE (a durable named listener) gets a durable consumer, as does // a STREAM source with a ConsumerGroup, or any SingleActiveConsumer. +// dlqSourceName is the conventional name of the source that reads a source's +// dead-letter queue. It has to match amqp091's byte for byte (setupDeadLetter +// there): that connector pre-declares a queue under this name whenever a source +// sets DeadLetterAddress, and clients attach to the dead letters by naming it — +// the arke integration suite's dead-letter tests do exactly that. The '.quorum' +// strip is amqp091's: a quorum queue's dead letters belong to the logical +// source, not to the queue-type suffix. +func dlqSourceName(sourceName string) string { + return strings.Replace(sourceName, ".quorum", "", 1) + ".dlq" +} + func durableName(source *pb.Source) string { if source.GetAutoDelete() || source.GetExclusive() || source.GetType() == pb.Source_TEMPORARY { return "" diff --git a/internal/provider/connectors/natsjs/helpers_test.go b/internal/provider/connectors/natsjs/helpers_test.go index 711f053..878beb0 100644 --- a/internal/provider/connectors/natsjs/helpers_test.go +++ b/internal/provider/connectors/natsjs/helpers_test.go @@ -360,6 +360,25 @@ func TestSubjectSubsumes(t *testing.T) { assert.False(t, cover("a.*", "*.b")) } +// TestDLQSourceName: the name has to match the amqp091 connector's byte for +// byte. It is the name clients attach to when reading a source's dead letters, +// so a mismatch means a reader that works on one broker finds nothing on the +// other — silently, since an absent queue just delivers no messages. +func TestDLQSourceName(t *testing.T) { + assert.Equal(t, "events.orders.listener.dlq", + dlqSourceName("events.orders.listener")) + + // The '.quorum' suffix is stripped: dead letters belong to the logical + // source, not to the queue type the other connector encodes in the name. + assert.Equal(t, "events.orders.listener.dlq", + dlqSourceName("events.orders.listener.quorum")) + + // Only the first occurrence, matching amqp091's strings.Replace(..., 1). + assert.Equal(t, "a.quorum.b.dlq", dlqSourceName("a.quorum.quorum.b")) + + assert.Equal(t, ".dlq", dlqSourceName("")) +} + func TestDurableName(t *testing.T) { q := func(name string) *pb.Source { return &pb.Source{Name: name, Address: &pb.Address{Name: "events.orders"}} diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 2827228..98e40c4 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -138,8 +138,22 @@ const ( // dead-letter. See deadLetterRetryDelay; override via // NATSJS_DEADLETTER_RETRY_DELAY. defaultDeadLetterRetryDelay = 5 * time.Second + // defaultDLQDeclareThreshold bounds how long a pre-declared dead-letter + // consumer survives with nothing reading it. See setupDeadLetter for why + // one is declared at all, and dlqDeclareThreshold for why this is bounded + // where amqp091's equivalent queue is not. Override via + // NATSJS_DLQ_DECLARE_TTL ("0" keeps it forever, matching amqp091). + defaultDLQDeclareThreshold = time.Hour ) +// dlqDurableMetadataKey marks a durable consumer as one setupDeadLetter +// pre-declared, and records the source whose dead letters it holds. Adoption +// (see adoptableDLQDurable) is gated on this marker rather than on the name +// alone: a transient source must never silently attach to some unrelated +// durable that merely happens to share its name, which would hand it another +// subscriber's pending messages and its stored ack position. +const dlqDurableMetadataKey = "arke_dead_letters_for" + // supportedSourceOptionsList intentionally matches the amqp091 connector so that // existing client Sources validate against the server unchanged. var supportedSourceOptionsList = []string{ @@ -461,6 +475,37 @@ func deadLetterRetryDelay() time.Duration { return defaultDeadLetterRetryDelay } +// dlqDeclareThreshold is the InactiveThreshold given to a pre-declared +// dead-letter consumer (setupDeadLetter). +// +// amqp091's equivalent — the queue setupDeadLetter declares there — has no +// expiry at all: it is not auto-delete, so once declared it persists until +// something removes it. Mirroring that exactly would leak here, because a +// source name is not always stable. Clients commonly mark a transient +// consumer by turning an Exclusive source into AutoDelete plus a fresh UUID +// (see durableName), so one using a transient source *with* a +// DeadLetterAddress would strand one never-expiring durable per connection, +// forever. Bounding the threshold trades the long tail of that parity (a DLQ +// reader attaching days later) for a leak that cannot run away: the reader +// still sees everything dead-lettered since the source was declared, provided +// it attaches within the window. +// +// NATSJS_DLQ_DECLARE_TTL overrides it; "0" disables expiry outright, which is +// full amqp091 parity for deployments whose source names are stable. +func dlqDeclareThreshold() time.Duration { + v := os.Getenv("NATSJS_DLQ_DECLARE_TTL") + if v == "" { + return defaultDLQDeclareThreshold + } + if v == "0" { + return 0 + } + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + return defaultDLQDeclareThreshold +} + func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { clientID, err := GetClientIdentifier(ctx) if err != nil { @@ -1193,8 +1238,33 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c // defaultInactiveThreshold (amqp091 defaults auto-delete/exclusive queues // to the same 5 minutes), durables never expire. consCfg.InactiveThreshold = expires - if durable := durableName(source); durable != "" { + durable := durableName(source) + adoptedDLQ := false + if durable == "" { + // A transient source is ordinarily an independent ephemeral reader, but + // the reader of a dead-letter queue is transient by convention while + // needing the opposite of a tail start: it exists to read what was + // dead-lettered before it attached. If setupDeadLetter pre-declared a + // durable under this source's name, that durable is the queue being + // named, so adopt it and inherit its position. + var marker map[string]string + durable, marker = adoptableDLQDurable(tctx, stream, source) + adoptedDLQ = durable != "" + // Carried forward so attaching does not erase the marker it matched on + // (see adoptableDLQDurable). + consCfg.Metadata = marker + } + if durable != "" { consCfg.Durable = durable + if adoptedDLQ && expires == 0 { + // Keep the bound setupDeadLetter declared. Adoption otherwise + // rewrites the pre-declared threshold to "never expire", so a + // reader that attached once and left would silently convert its + // dead-letter queue into permanent server state — the leak + // dlqDeclareThreshold exists to bound. An explicit Expires still + // wins, as it does for every other source. + consCfg.InactiveThreshold = dlqDeclareThreshold() + } if source.GetSingleActiveConsumer() { // RabbitMQ's single-active-consumer delivers to one consumer at a // time — the point is ordered processing across competing @@ -1268,6 +1338,12 @@ func (p *natsjsProvider) Subscribe(ctx context.Context, source *pb.Source, out c } } + // Pre-declare the dead-letter reader's consumer now that this source's own + // topology exists, matching where amqp091 does it (after its queue and + // binding, before the DeclareOnly return) so a declare-only subscribe + // establishes the same topology on both connectors. + p.setupDeadLetter(tctx, bd, source) + // DeclareOnly: topology established, do not consume. An ephemeral consumer // created only to validate that topology is garbage the moment we return — // its server-generated name is never seen again — so delete it eagerly @@ -1909,6 +1985,122 @@ func (p *natsjsProvider) Retry(ctx context.Context, _ *pb.Source, uuid string, d return nil } +// setupDeadLetter pre-declares the consumer that holds a source's dead letters, +// mirroring the amqp091 connector's setupDeadLetter. Best-effort: a failure +// here leaves dead-lettering itself working (DeadLetter ensures the stream on +// its own) and must not fail the subscribe, which is how amqp091 treats it too. +// +// Why this exists at all. On RabbitMQ the pass is not a broker property: arke +// declares a queue named dlqSourceName(source) and binds it to the DLX the +// moment a source declares a DeadLetterAddress, before any message can be +// dead-lettered. From then on that queue accumulates, so a consumer attaching +// later — after the failures, which is the normal shape for a DLQ reader — +// finds them waiting. natsjs had no equivalent: it ensured the DLQ stream +// lazily inside DeadLetter, and a late reader (Source_TEMPORARY, hence +// ephemeral, hence DeliverNew) started at the stream's tail and saw an empty +// queue while its messages sat in the stream just behind it. +// +// The faithful start position is therefore neither "tail at attach" (what we +// did: misses everything) nor DeliverAll (replays up to MaxAge of history the +// RabbitMQ queue never held). It is the tail *as of declare time*, which is +// exactly what a durable created here with DeliverNew records, and what the +// reader inherits by adopting it (adoptableDLQDurable). +func (p *natsjsProvider) setupDeadLetter(ctx context.Context, bd *natsBrokerDetails, origSource *pb.Source) { + opts := origSource.GetOptions() + dla := opts["DeadLetterAddress"] + // Absent or empty are both no-ops. Empty is not hypothetical: amqp091's own + // Retry builds a source with DeadLetterAddress "" to mean "expire back to + // the default exchange", and nothing should be declared for that. + if dla == "" { + return + } + durable := streamNameFor(dlqSourceName(origSource.GetName())) + + streamName, serr := p.ensureStream(ctx, bd, dla) + if serr != nil { + util.Logger.Warn( + "natsjs: source {0} declares dead-letter address {1} but its stream could not be ensured ({2}); "+ + "dead letters published before a reader attaches will not be visible to it", + origSource.GetName(), dla, serr.Error()) + return + } + stream, serr := bd.js.Stream(ctx, streamName) + if serr != nil { + util.Logger.Warn("natsjs: open dead-letter stream {0} for source {1}: {2}", + streamName, origSource.GetName(), serr.Error()) + return + } + // Already declared: skip the create. This saves a management round trip on + // every subscribe of a source with a dead-letter address; it is not what + // protects the position a queue has been accumulating. Nothing here could + // move that anyway — after creation a durable's position is its stored ack + // floor, not its declared DeliverPolicy, so re-asserting the same config is + // a no-op for it, and a genuinely different start position is refused + // outright (err 10012, which Subscribe absorbs separately). Creating rather + // than updating below keeps that true even if this check is removed. + if _, cerr := stream.Consumer(ctx, durable); cerr == nil { + return + } + // Capture the whole dead-letter address, not just where DeadLetterSubject + // says this source's copies will land. Narrowing here would be false + // precision: what a reader ultimately sees is decided by the filter IT + // asks for, because adopting rewrites the filter (mutable) while + // inheriting the position (not). The pre-declared filter therefore only + // affects num_pending until someone attaches — and a source with no + // DeadLetterSubject cannot be narrowed anyway, since DeadLetter then + // preserves each message's own routing key. + if _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: durable, + FilterSubjects: wholeAddressSubjects(dla), + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverNewPolicy, + AckWait: ackWait(), + InactiveThreshold: dlqDeclareThreshold(), + Metadata: map[string]string{dlqDurableMetadataKey: origSource.GetName()}, + }); cerr != nil { + util.Logger.Warn( + "natsjs: pre-declare of dead-letter consumer {0} for source {1} failed ({2}); "+ + "dead-lettering still works, but a reader attaching later may not see earlier dead letters", + durable, origSource.GetName(), cerr.Error()) + } +} + +// adoptableDLQDurable returns the pre-declared dead-letter durable this source +// is the reader for, or "" if there is none. +// +// A DLQ reader is conventionally transient (the suite's dead-letter tests use +// Source_TEMPORARY and AutoDelete), which would otherwise make it ephemeral and +// start it at the stream tail — after the dead letters it exists to read. When +// setupDeadLetter has already declared a durable under this source's name, that +// durable *is* the queue the reader is naming, so attach to it and inherit the +// position it has been holding since the source was declared. +// +// Adoption is deliberately gated on the marker metadata, not on the name alone: +// name-only matching would let any transient source hijack an unrelated durable +// that happens to share its name, taking over another subscriber's pending +// messages and ack position. A lookup miss of any kind means no adoption. +// +// It returns the metadata to carry forward alongside the name. That is not +// cosmetic: attaching re-asserts the consumer's whole config, and a config +// without the marker REPLACES the stored metadata and erases it — after which +// the durable is unadoptable and the next reader of the same queue silently +// falls back to an ephemeral at the tail, which is the exact failure this +// whole mechanism exists to prevent. setupDeadLetter cannot repair that later: +// it leaves an existing consumer alone precisely so it never moves a position +// that has been accumulating. +func adoptableDLQDurable(ctx context.Context, stream jetstream.Stream, source *pb.Source) (string, map[string]string) { + durable := streamNameFor(source.GetName()) + cons, err := stream.Consumer(ctx, durable) + if err != nil { + return "", nil + } + marker, ok := cons.CachedInfo().Config.Metadata[dlqDurableMetadataKey] + if !ok { + return "", nil + } + return durable, map[string]string{dlqDurableMetadataKey: marker} +} + // DeadLetter publishes the message to the configured dead-letter subject (there // is no native DLX in JetStream) and terminates redelivery. // diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index bd2368f..1473bbc 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -483,6 +483,287 @@ func TestDeadLetterPreservesRoutingKeyRoutedFromParent(t *testing.T) { // original. It returns an error and leaves the message in flight, so the // caller's fallback nack still resolves the uuid and the message is // redelivered — the failure mode is retry, not silent loss. +// TestLateDLQReaderSeesEarlierDeadLetters: a dead-letter reader that attaches +// after the failures still receives them. The reader is transient by +// convention (the arke integration suite's dead-letter tests use +// Source_TEMPORARY), which makes it ephemeral and starts it at the stream tail +// — behind the very messages it exists to read. amqp091 passes this because +// arke declares the DLQ queue when the *source* declares its +// DeadLetterAddress, so it accumulates from that moment; setupDeadLetter +// mirrors it with a pre-declared durable the reader adopts. +// +// Deliberately cross-connection: the reader is a different client from the +// source that declared the DLA, so no per-connection state can carry the +// position (see the connector's per-connection memos — knownStreams, +// durableFilters). Only server-side topology can. +func TestLateDLQReaderSeesEarlierDeadLetters(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.late", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.late.consumer", "events.late", "job") + src.Options["DeadLetterAddress"] = "events.late.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // Only now does the reader attach, on its own connection, naming the + // conventional DLQ source name and asking for no Offset — it must not need + // one, exactly as it does not on RabbitMQ. + p2, ctx2 := connectClient(t, s) + readerOut := make(chan *pb.Message, 1) + reader := &pb.Source{ + Name: dlqSourceName("events.late.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.late.dlq", Subjects: []string{"failed"}}, + } + readCtx, readCancel := context.WithCancel(ctx2) + defer readCancel() + go p2.Subscribe(readCtx, reader, readerOut) + + got := recv(t, readerOut) + assert.Equal(t, "poison", string(got.GetBody())) +} + +// TestDLQDurableAdoptableByASecondReader: attaching to a pre-declared +// dead-letter durable re-asserts its config, and a config without the marker +// metadata would REPLACE the stored metadata and erase it — leaving the +// durable unadoptable, so the next reader of the same queue would silently +// fall back to an ephemeral at the tail. Found in the arke integration suite, +// where TestDeadLettering and TestDeadLetteringReject share a source name: +// the first test's reader wiped the marker and the second saw nothing. +func TestDLQDurableAdoptableByASecondReader(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.twice", Subjects: []string{"job"}} + out := make(chan *pb.Message, 2) + src := queueSource("events.twice.consumer", "events.twice", "job") + src.Options["DeadLetterAddress"] = "events.twice.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + reader := func() *pb.Source { + return &pb.Source{ + Name: dlqSourceName("events.twice.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.twice.dlq", Subjects: []string{"failed"}}, + } + } + + // First reader: attaches, drains one dead letter, goes away. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("first")})) + first := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, first.GetUuid())) + + firstOut := make(chan *pb.Message, 1) + firstCtx, firstCancel := context.WithCancel(ctx) + go p.Subscribe(firstCtx, reader(), firstOut) + got := recv(t, firstOut) + require.Equal(t, "first", string(got.GetBody())) + require.Nil(t, p.Ack(ctx, got.GetUuid())) + firstCancel() + + // Second reader, after another failure: must still adopt the same durable. + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("second")})) + next := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, next.GetUuid())) + + secondOut := make(chan *pb.Message, 1) + secondCtx, secondCancel := context.WithCancel(ctx) + defer secondCancel() + go p.Subscribe(secondCtx, reader(), secondOut) + assert.Equal(t, "second", string(recv(t, secondOut).GetBody())) +} + +// TestLateDLQReaderNoDeadLetterSubject covers the other filter branch: with no +// DeadLetterSubject the dead-lettered copy keeps its ORIGINAL routing key, +// which is per-message and unknowable when the queue is pre-declared, so +// setupDeadLetter captures the whole dead-letter address instead. A reader +// binding by that original key must still find it. This is the shape a source +// setting only DeadLetterAddress produces — the common one. +func TestLateDLQReaderNoDeadLetterSubject(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.nosub", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + + out := make(chan *pb.Message, 1) + src := queueSource("events.nosub.consumer", "events.nosub", "job") + src.Options["DeadLetterAddress"] = "events.nosub.dlq" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + readerOut := make(chan *pb.Message, 1) + reader := &pb.Source{ + Name: dlqSourceName("events.nosub.consumer"), + Type: pb.Source_TEMPORARY, + // Bound by the ORIGINAL routing key, which is what the copy carries. + Address: &pb.Address{Name: "events.nosub.dlq", Subjects: []string{"job"}}, + } + readCtx, readCancel := context.WithCancel(ctx) + defer readCancel() + go p.Subscribe(readCtx, reader, readerOut) + + assert.Equal(t, "poison", string(recv(t, readerOut).GetBody())) +} + +// TestSetupDeadLetterLeavesExistingDurableAlone: re-declaring a source must not +// disturb a dead-letter queue that has been accumulating. setupDeadLetter +// returns early when the durable exists precisely so it never moves a position +// — re-asserting it at DeliverNew would silently discard every dead letter +// collected since the first declare, and clients re-subscribe routinely +// (reconnects, rollouts), so this is a common path, not an edge case. +func TestSetupDeadLetterLeavesExistingDurableAlone(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + addr := &pb.Address{Name: "events.redeclare", Subjects: []string{"job"}} + out := make(chan *pb.Message, 2) + src := queueSource("events.redeclare.consumer", "events.redeclare", "job") + src.Options["DeadLetterAddress"] = "events.redeclare.dlq" + src.Options["DeadLetterSubject"] = "failed" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: addr, Body: []byte("poison")})) + m := recv(t, out) + require.Nil(t, p.DeadLetter(ctx, src, m.GetUuid())) + + // Re-declare the same source, as a reconnecting client does. If this moved + // the pre-declared durable's start position, the dead letter above would + // become invisible. + againCtx, againCancel := context.WithCancel(ctx) + go p.Subscribe(againCtx, src, make(chan *pb.Message, 1)) + bd := p.getBrokerDetailsByIdentifier("test-client") + require.Eventually(t, func() bool { + st, serr := bd.js.Stream(ctx, streamNameFor("events.redeclare.dlq")) + if serr != nil { + return false + } + info, ierr := st.Info(ctx) + return ierr == nil && info.State.Msgs == 1 + }, 10*time.Second, 20*time.Millisecond, "dead letter never stored") + againCancel() + + readerOut := make(chan *pb.Message, 1) + readCtx, readCancel := context.WithCancel(ctx) + defer readCancel() + go p.Subscribe(readCtx, &pb.Source{ + Name: dlqSourceName("events.redeclare.consumer"), + Type: pb.Source_TEMPORARY, + Address: &pb.Address{Name: "events.redeclare.dlq", Subjects: []string{"failed"}}, + }, readerOut) + + assert.Equal(t, "poison", string(recv(t, readerOut).GetBody())) +} + +// TestSetupDeadLetterEmptyAddressDeclaresNothing: an empty DeadLetterAddress +// must declare no topology at all. Not hypothetical — the amqp091 connector's +// own Retry builds a source with DeadLetterAddress "" to mean "expire back to +// the default exchange", and an empty address maps to the degenerate stream +// name "arke_". +func TestSetupDeadLetterEmptyAddressDeclaresNothing(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + out := make(chan *pb.Message, 1) + src := queueSource("events.emptydla.consumer", "events.emptydla", "job") + src.Options["DeadLetterAddress"] = "" + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + require.Nil(t, p.PublishOne(ctx, &pb.Message{ + Address: &pb.Address{Name: "events.emptydla", Subjects: []string{"job"}}, + Body: []byte("hello"), + })) + require.Equal(t, "hello", string(recv(t, out).GetBody())) + + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.Stream(ctx, streamNameFor("")) + assert.ErrorIs(t, serr, jetstream.ErrStreamNotFound, + "an empty DeadLetterAddress must not create a stream") +} + +func TestDLQDeclareTTLFromEnv(t *testing.T) { + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "10m") + assert.Equal(t, 10*time.Minute, dlqDeclareThreshold()) + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "bad") + assert.Equal(t, defaultDLQDeclareThreshold, dlqDeclareThreshold()) + // "0" is meaningful here, unlike the other thresholds: it opts out of + // expiry entirely, which is exact amqp091 parity. + t.Setenv("NATSJS_DLQ_DECLARE_TTL", "0") + assert.Equal(t, time.Duration(0), dlqDeclareThreshold()) +} + +// TestTransientSourceDoesNotAdoptUnmarkedDurable pins the gate on adoption: a +// transient source whose name merely collides with an existing durable must +// stay ephemeral. Adopting on name alone would hand it another subscriber's +// pending messages and stored ack position. +func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { + s := runJetStreamServer(t) + p, ctx := connectClient(t, s) + + // A durable with the name a transient source would look for, but none of + // arke's dead-letter marker metadata. + bd := p.getBrokerDetailsByIdentifier("test-client") + _, serr := bd.js.CreateStream(ctx, jetstream.StreamConfig{ + Name: streamNameFor("events.collide"), + Subjects: streamSubjectsFor("events.collide"), + }) + require.NoError(t, serr) + stream, serr := bd.js.Stream(ctx, streamNameFor("events.collide")) + require.NoError(t, serr) + _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: streamNameFor("events.collide.reader"), + AckPolicy: jetstream.AckExplicitPolicy, + }) + require.NoError(t, cerr) + + srcAddr := &pb.Address{Name: "events.collide", Subjects: []string{"job"}} + require.Nil(t, p.PublishOne(ctx, &pb.Message{Address: srcAddr, Body: []byte("hello")})) + + out := make(chan *pb.Message, 1) + src := &pb.Source{ + Name: "events.collide.reader", + Type: pb.Source_TEMPORARY, + Address: srcAddr, + // Read from the start, so receipt does not race consumer setup. The + // discriminator here is the untouched durable below, not the delivery. + Options: map[string]string{"Offset": "first"}, + } + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go p.Subscribe(subCtx, src, out) + + got := recv(t, out) + require.Equal(t, "hello", string(got.GetBody())) + + // The pre-existing durable is untouched: the subscribe went to an ephemeral + // consumer of its own rather than adopting it. + existing, ierr := stream.Consumer(ctx, streamNameFor("events.collide.reader")) + require.NoError(t, ierr) + info, ierr := existing.Info(ctx) + require.NoError(t, ierr) + assert.Zero(t, info.Delivered.Consumer, "an unmarked durable must not have been consumed from") +} + func TestDeadLetterFailureKeepsMessage(t *testing.T) { // The fallback nack of a failed dead-letter is paced (see // deadLetterRetryDelay); shorten it so the test does not wait it out. From 9bf6609c4dfa23709cb86ab7f3ec50dac099500f Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 22 Jul 2026 17:13:26 -0400 Subject: [PATCH 76/77] perf: only look up a dead-letter durable for a source that could have one Adoption asked the server for a consumer on every transient subscribe, and for all but a dead-letter reader the answer is always "consumer not found". That is a management round trip per subscribe, and because the answer is an error response it also inflates the JetStream API error counter that deployments watch as a health signal -- worst during the reconnect storm after a restart, when every transient consumer re-subscribes at once and that signal matters most. Check the source name first. This is exact rather than a heuristic: the only durables pre-declared here are named streamNameFor(.dlq) and streamNameFor is injective, so a match requires the reader's own name to carry the suffix. No lookup is skipped that could have hit. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/helpers.go | 7 ++++++- internal/provider/connectors/natsjs/natsjs.go | 13 +++++++++++++ internal/provider/connectors/natsjs/natsjs_test.go | 11 ++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/provider/connectors/natsjs/helpers.go b/internal/provider/connectors/natsjs/helpers.go index 0a8d27f..378506d 100644 --- a/internal/provider/connectors/natsjs/helpers.go +++ b/internal/provider/connectors/natsjs/helpers.go @@ -599,9 +599,14 @@ func subjectSubsumes(wide, narrow string) bool { // strip is amqp091's: a quorum queue's dead letters belong to the logical // source, not to the queue-type suffix. func dlqSourceName(sourceName string) string { - return strings.Replace(sourceName, ".quorum", "", 1) + ".dlq" + return strings.Replace(sourceName, ".quorum", "", 1) + dlqSourceNameSuffix } +// dlqSourceNameSuffix is what dlqSourceName appends. Named because the +// adoption path tests for it to skip a server lookup for every source that +// cannot possibly be a dead-letter reader (see adoptableDLQDurable). +const dlqSourceNameSuffix = ".dlq" + func durableName(source *pb.Source) string { if source.GetAutoDelete() || source.GetExclusive() || source.GetType() == pb.Source_TEMPORARY { return "" diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index 98e40c4..e1edc8e 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -2089,6 +2089,19 @@ func (p *natsjsProvider) setupDeadLetter(ctx context.Context, bd *natsBrokerDeta // it leaves an existing consumer alone precisely so it never moves a position // that has been accumulating. func adoptableDLQDurable(ctx context.Context, stream jetstream.Stream, source *pb.Source) (string, map[string]string) { + // Only a source named like a dead-letter reader can match one, so check the + // name before asking the server. This is exact, not a heuristic: the only + // durables setupDeadLetter creates are named streamNameFor(.dlq), + // and streamNameFor is injective, so a match requires the reader's own name + // to end in the suffix. Skipping the lookup matters because every OTHER + // transient subscribe would otherwise pay a management round trip whose + // answer is always "consumer not found" — and that answer is an error + // response, so it also inflates the JetStream API error counter that + // deployments watch as a health signal, worst during the reconnect storm + // after a restart when every transient consumer re-subscribes at once. + if !strings.HasSuffix(source.GetName(), dlqSourceNameSuffix) { + return "", nil + } durable := streamNameFor(source.GetName()) cons, err := stream.Consumer(ctx, durable) if err != nil { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 1473bbc..9985a1c 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -716,6 +716,11 @@ func TestDLQDeclareTTLFromEnv(t *testing.T) { // transient source whose name merely collides with an existing durable must // stay ephemeral. Adopting on name alone would hand it another subscriber's // pending messages and stored ack position. +// +// The source is named with the dead-letter suffix deliberately, so it clears +// the cheap name check in adoptableDLQDurable and actually reaches the marker +// check this test is about. Without the suffix it would short-circuit earlier +// and pass for the wrong reason. func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { s := runJetStreamServer(t) p, ctx := connectClient(t, s) @@ -731,7 +736,7 @@ func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { stream, serr := bd.js.Stream(ctx, streamNameFor("events.collide")) require.NoError(t, serr) _, cerr := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: streamNameFor("events.collide.reader"), + Durable: streamNameFor("events.collide.reader.dlq"), AckPolicy: jetstream.AckExplicitPolicy, }) require.NoError(t, cerr) @@ -741,7 +746,7 @@ func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { out := make(chan *pb.Message, 1) src := &pb.Source{ - Name: "events.collide.reader", + Name: "events.collide.reader.dlq", Type: pb.Source_TEMPORARY, Address: srcAddr, // Read from the start, so receipt does not race consumer setup. The @@ -757,7 +762,7 @@ func TestTransientSourceDoesNotAdoptUnmarkedDurable(t *testing.T) { // The pre-existing durable is untouched: the subscribe went to an ephemeral // consumer of its own rather than adopting it. - existing, ierr := stream.Consumer(ctx, streamNameFor("events.collide.reader")) + existing, ierr := stream.Consumer(ctx, streamNameFor("events.collide.reader.dlq")) require.NoError(t, ierr) info, ierr := existing.Info(ctx) require.NoError(t, ierr) From 00a70df36e150cd0efa7a5ca73e9a3d4b27ac101 Mon Sep 17 00:00:00 2001 From: Michael Otteni Date: Wed, 22 Jul 2026 21:20:01 -0400 Subject: [PATCH 77/77] fix: expire a pre-declared dead-letter queue with its messages, not before The pre-declared dead-letter consumer expired after a fixed hour with nothing reading it, while its stream retains messages for MaxAge (72h by default). That left a window where the dead letters were still stored but the queue holding them was gone -- and a reader attaching in that window falls back to an ephemeral at the stream tail and silently sees nothing, which is the exact failure pre-declaring it is meant to prevent. The window was not an edge case: a dead-letter reader attaches AFTER the failures, because finding out what broke is why it is attaching. An hour is far shorter than the gap between "it broke overnight" and "someone looked", so the fixed default missed the case that motivates the feature. Default to the stream's own retention instead, so the queue never expires while there is still something in it to read. This closes the window by construction rather than by guessing a duration, and it tracks whatever retention a deployment actually configures. A stream set to retain forever now yields a queue that never expires, consistently. Bounding it at all still matters -- an unbounded threshold strands one permanent durable per connection for clients that name transient consumers with a fresh UUID -- so this trades only the parity past the point where the messages themselves are gone. Signed-off-by: Michael Otteni --- internal/provider/connectors/natsjs/natsjs.go | 37 +++++++++---------- .../provider/connectors/natsjs/natsjs_test.go | 16 +++++++- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/internal/provider/connectors/natsjs/natsjs.go b/internal/provider/connectors/natsjs/natsjs.go index e1edc8e..4c96904 100644 --- a/internal/provider/connectors/natsjs/natsjs.go +++ b/internal/provider/connectors/natsjs/natsjs.go @@ -138,12 +138,6 @@ const ( // dead-letter. See deadLetterRetryDelay; override via // NATSJS_DEADLETTER_RETRY_DELAY. defaultDeadLetterRetryDelay = 5 * time.Second - // defaultDLQDeclareThreshold bounds how long a pre-declared dead-letter - // consumer survives with nothing reading it. See setupDeadLetter for why - // one is declared at all, and dlqDeclareThreshold for why this is bounded - // where amqp091's equivalent queue is not. Override via - // NATSJS_DLQ_DECLARE_TTL ("0" keeps it forever, matching amqp091). - defaultDLQDeclareThreshold = time.Hour ) // dlqDurableMetadataKey marks a durable consumer as one setupDeadLetter @@ -478,24 +472,29 @@ func deadLetterRetryDelay() time.Duration { // dlqDeclareThreshold is the InactiveThreshold given to a pre-declared // dead-letter consumer (setupDeadLetter). // -// amqp091's equivalent — the queue setupDeadLetter declares there — has no -// expiry at all: it is not auto-delete, so once declared it persists until -// something removes it. Mirroring that exactly would leak here, because a -// source name is not always stable. Clients commonly mark a transient -// consumer by turning an Exclusive source into AutoDelete plus a fresh UUID -// (see durableName), so one using a transient source *with* a -// DeadLetterAddress would strand one never-expiring durable per connection, -// forever. Bounding the threshold trades the long tail of that parity (a DLQ -// reader attaching days later) for a leak that cannot run away: the reader -// still sees everything dead-lettered since the source was declared, provided -// it attaches within the window. +// It defaults to the dead-letter stream's own retention (streamMaxAge), so the +// queue never expires while there is still something in it to read. That is +// the property that matters: a dead-letter reader attaches AFTER the failures +// — finding out what broke is why it is attaching — so any window where the +// messages are still retained but their queue has already gone is a window +// where the reader silently sees nothing, which is the exact bug +// setupDeadLetter exists to fix. Tying the two together closes it by +// construction rather than by guessing a duration. A stream configured for +// unbounded retention (MaxAge 0) yields 0 here too: no expiry, consistently. +// +// amqp091's equivalent queue has no expiry at all, so this is still a +// divergence, but only past the point where the messages themselves are gone. +// Bounding it at all is what stops a leak: a source name is not always stable +// — clients commonly mark a transient consumer by turning an Exclusive source +// into AutoDelete plus a fresh UUID (see durableName) — so an unbounded +// threshold would strand one permanent durable per connection. // // NATSJS_DLQ_DECLARE_TTL overrides it; "0" disables expiry outright, which is // full amqp091 parity for deployments whose source names are stable. func dlqDeclareThreshold() time.Duration { v := os.Getenv("NATSJS_DLQ_DECLARE_TTL") if v == "" { - return defaultDLQDeclareThreshold + return streamMaxAge() } if v == "0" { return 0 @@ -503,7 +502,7 @@ func dlqDeclareThreshold() time.Duration { if d, err := time.ParseDuration(v); err == nil && d > 0 { return d } - return defaultDLQDeclareThreshold + return streamMaxAge() } func (p *natsjsProvider) getBrokerDetails(ctx context.Context) (*natsBrokerDetails, error) { diff --git a/internal/provider/connectors/natsjs/natsjs_test.go b/internal/provider/connectors/natsjs/natsjs_test.go index 9985a1c..5f09d44 100644 --- a/internal/provider/connectors/natsjs/natsjs_test.go +++ b/internal/provider/connectors/natsjs/natsjs_test.go @@ -704,10 +704,24 @@ func TestSetupDeadLetterEmptyAddressDeclaresNothing(t *testing.T) { func TestDLQDeclareTTLFromEnv(t *testing.T) { t.Setenv("NATSJS_DLQ_DECLARE_TTL", "10m") assert.Equal(t, 10*time.Minute, dlqDeclareThreshold()) + + // Unset and unparseable both fall back to the dead-letter stream's own + // retention, so the queue outlives the messages in it instead of expiring + // at some guessed duration while they are still there to read. t.Setenv("NATSJS_DLQ_DECLARE_TTL", "bad") - assert.Equal(t, defaultDLQDeclareThreshold, dlqDeclareThreshold()) + assert.Equal(t, defaultStreamMaxAge, dlqDeclareThreshold()) + t.Setenv("NATSJS_STREAM_MAX_AGE", "6h") + assert.Equal(t, 6*time.Hour, dlqDeclareThreshold(), + "the pre-declared queue must track the retention actually configured") + + // A stream retaining forever gives a queue that never expires, so there is + // still no window where messages outlive the queue holding them. + t.Setenv("NATSJS_STREAM_MAX_AGE", "0") + assert.Equal(t, time.Duration(0), dlqDeclareThreshold()) + // "0" is meaningful here, unlike the other thresholds: it opts out of // expiry entirely, which is exact amqp091 parity. + t.Setenv("NATSJS_STREAM_MAX_AGE", "72h") t.Setenv("NATSJS_DLQ_DECLARE_TTL", "0") assert.Equal(t, time.Duration(0), dlqDeclareThreshold()) }